diff --git "a/wandb/run-20221025_092127-29xi74uq/logs/debug-internal.log" "b/wandb/run-20221025_092127-29xi74uq/logs/debug-internal.log" --- "a/wandb/run-20221025_092127-29xi74uq/logs/debug-internal.log" +++ "b/wandb/run-20221025_092127-29xi74uq/logs/debug-internal.log" @@ -1,68537 +1,3 @@ -2022-10-25 09:21:27,949 INFO MainThread:2194005 [internal.py:wandb_internal():90] W&B internal server running at pid: 2194005, started at: 2022-10-25 09:21:27.949156 -2022-10-25 09:21:27,952 INFO WriterThread:2194005 [datastore.py:open_for_write():75] open: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/run-29xi74uq.wandb -2022-10-25 09:21:27,954 DEBUG SenderThread:2194005 [sender.py:send():232] send: header -2022-10-25 09:21:27,954 DEBUG SenderThread:2194005 [sender.py:send():232] send: run -2022-10-25 09:21:28,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: check_version -2022-10-25 09:21:28,233 INFO SenderThread:2194005 [dir_watcher.py:__init__():166] watching files in: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files -2022-10-25 09:21:28,234 INFO SenderThread:2194005 [sender.py:_start_run_threads():811] run started: 29xi74uq with start time 1666682487 -2022-10-25 09:21:28,234 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 09:21:28,234 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 09:21:28,234 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: check_version -2022-10-25 09:21:28,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: run_start -2022-10-25 09:21:28,390 DEBUG HandlerThread:2194005 [meta.py:__init__():35] meta init -2022-10-25 09:21:28,390 DEBUG HandlerThread:2194005 [meta.py:__init__():49] meta init done -2022-10-25 09:21:28,390 DEBUG HandlerThread:2194005 [meta.py:probe():209] probe -2022-10-25 09:21:28,392 DEBUG HandlerThread:2194005 [meta.py:_setup_git():199] setup git -2022-10-25 09:21:28,412 DEBUG HandlerThread:2194005 [meta.py:_setup_git():206] setup git done -2022-10-25 09:21:28,412 DEBUG HandlerThread:2194005 [meta.py:_save_pip():53] save pip -2022-10-25 09:21:28,413 DEBUG HandlerThread:2194005 [meta.py:_save_pip():67] save pip done -2022-10-25 09:21:28,413 DEBUG HandlerThread:2194005 [meta.py:probe():247] probe done -2022-10-25 09:21:28,420 DEBUG SenderThread:2194005 [sender.py:send():232] send: files -2022-10-25 09:21:28,421 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-metadata.json with policy now -2022-10-25 09:21:28,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:21:28,431 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:21:28,653 DEBUG SenderThread:2194005 [sender.py:send():232] send: telemetry -2022-10-25 09:21:28,654 DEBUG SenderThread:2194005 [sender.py:send():232] send: config -2022-10-25 09:21:28,656 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 09:21:28,656 DEBUG SenderThread:2194005 [sender.py:send():232] send: telemetry -2022-10-25 09:21:28,656 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 09:21:28,656 WARNING SenderThread:2194005 [sender.py:send_metric():904] Seen metric with glob (shouldn't happen) -2022-10-25 09:21:28,656 DEBUG SenderThread:2194005 [sender.py:send():232] send: telemetry -2022-10-25 09:21:29,068 INFO Thread-11 :2194005 [upload_job.py:push():137] Uploaded file /tmp/tmp92typzyqwandb/3773v7qx-wandb-metadata.json -2022-10-25 09:21:29,237 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_created():213] file/dir created: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 09:21:29,237 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_created():213] file/dir created: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/requirements.txt -2022-10-25 09:21:29,238 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_created():213] file/dir created: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 09:21:29,238 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_created():213] file/dir created: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-metadata.json -2022-10-25 09:21:31,235 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 09:21:43,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:21:43,889 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:21:56,466 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:21:59,247 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/config.yaml -2022-10-25 09:21:59,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:21:59,302 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:22:14,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:22:14,633 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:22:26,551 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:22:29,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:22:29,966 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:22:45,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:22:45,413 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:22:56,629 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:23:00,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:23:00,749 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:23:16,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:23:16,238 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:23:26,707 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:23:31,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:23:31,524 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:23:46,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:23:46,879 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:23:56,782 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:24:02,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:24:02,117 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:24:17,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:24:17,445 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:24:26,859 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:24:32,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:24:32,814 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:24:48,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:24:48,069 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:24:56,935 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:25:03,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:25:03,342 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:25:18,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:25:18,594 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:25:27,013 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:25:33,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:25:33,951 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:25:49,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:25:49,271 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:25:57,092 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:26:04,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:26:04,561 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:26:19,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:26:19,856 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:26:27,169 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:26:35,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:26:35,214 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:26:50,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:26:50,564 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:26:57,247 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:27:05,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:27:05,867 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:27:21,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:27:21,233 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:27:27,329 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:27:36,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:27:36,505 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:27:51,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:27:51,872 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:27:57,407 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:28:07,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:28:07,147 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:28:22,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:28:22,514 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:28:27,485 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:28:37,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:28:37,812 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:28:53,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:28:53,148 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:28:57,563 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:29:08,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:29:08,440 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:29:23,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:29:23,725 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:29:27,642 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:29:39,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:29:39,058 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:29:54,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:29:54,318 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:29:57,720 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:30:09,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:30:09,675 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:30:25,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:30:25,026 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:30:27,798 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:30:40,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:30:40,314 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:30:55,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:30:55,616 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:30:57,875 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:31:11,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:31:11,044 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:31:26,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:31:26,424 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:31:27,954 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:31:41,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:31:41,815 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:31:57,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:31:57,153 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:31:58,031 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:32:12,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:32:12,497 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:32:27,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:32:27,804 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:32:28,107 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:32:43,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:32:43,138 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:32:58,185 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:32:58,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:32:58,421 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:33:13,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:33:13,779 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:33:28,261 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:33:29,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:33:29,080 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:33:44,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:33:44,405 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:33:58,338 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:33:59,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:33:59,816 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:34:15,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:34:15,110 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:34:28,415 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:34:30,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:34:30,492 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:34:45,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:34:45,778 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:34:58,492 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:35:01,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:35:01,197 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:35:16,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:35:16,512 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:35:28,572 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:35:31,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:35:31,869 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:35:47,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:35:47,256 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:35:58,652 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:36:02,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:36:02,583 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:36:17,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:36:17,830 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:36:28,732 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:36:33,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:36:33,174 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:36:48,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:36:48,429 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:36:58,809 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:37:03,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:37:03,729 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:37:19,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:37:19,053 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:37:28,885 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:37:34,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:37:34,440 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:37:51,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:37:51,533 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:37:58,962 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:38:06,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:38:06,836 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:38:22,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:38:22,232 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:38:29,039 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:38:37,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:38:37,575 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:38:52,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:38:52,881 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:38:59,112 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:39:08,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:39:08,200 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:39:23,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:39:23,594 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:39:29,183 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:39:38,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:39:38,990 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:39:54,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:39:54,260 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:39:59,258 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:40:09,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:40:09,587 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:40:24,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:40:24,929 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:40:29,330 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:40:40,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:40:40,365 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:40:55,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:40:55,695 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:40:59,410 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:41:10,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:41:10,992 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:41:26,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:41:26,413 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:41:29,487 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:41:41,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:41:41,728 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:41:57,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:41:57,219 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:41:59,566 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:42:12,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:42:12,592 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:42:28,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:42:28,016 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:42:29,642 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:42:43,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:42:43,295 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:42:58,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:42:58,710 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:42:59,718 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:43:14,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:43:14,077 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:43:29,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:43:29,470 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:43:29,817 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:43:44,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:43:44,948 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:43:59,872 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:44:00,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:44:00,309 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:44:15,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:44:15,575 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:44:29,948 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:44:30,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:44:30,940 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:44:46,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:44:46,305 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:45:00,025 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:45:01,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:45:01,689 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:45:17,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:45:17,123 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:45:30,102 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:45:32,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:45:32,396 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:45:47,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:45:47,815 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:46:00,179 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:46:03,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:46:03,153 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:46:18,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:46:18,616 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:46:30,256 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:46:33,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:46:33,940 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:46:49,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:46:49,362 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:47:00,335 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:47:04,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:47:04,654 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:47:20,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:47:20,086 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:47:30,413 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:47:35,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:47:35,477 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:47:50,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:47:50,858 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:48:00,492 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:48:06,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:48:06,280 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:48:21,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:48:21,585 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:48:30,570 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:48:37,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:48:37,029 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:48:52,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:48:52,369 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:49:00,647 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:49:07,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:49:07,627 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:49:22,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:49:22,963 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:49:30,726 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:49:38,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:49:38,344 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:49:53,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:49:53,815 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:50:00,804 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:50:09,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:50:09,129 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:50:24,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:50:24,409 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:50:30,883 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:50:39,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:50:39,798 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:50:55,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:50:55,200 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:51:00,963 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:51:10,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:51:10,612 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:51:25,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:51:25,909 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:51:31,042 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:51:41,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:51:41,327 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:51:56,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:51:56,784 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:52:01,119 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:52:12,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:52:12,115 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:52:27,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:52:27,566 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:52:31,195 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:52:42,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:52:42,895 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:52:58,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:52:58,218 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:53:01,272 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:53:13,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:53:13,605 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:53:29,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:53:29,050 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:53:31,350 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:53:44,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:53:44,508 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:53:59,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:53:59,853 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:54:01,428 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:54:15,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:54:15,212 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:54:30,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:54:30,616 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:54:31,503 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:54:45,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:54:45,940 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:55:01,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:55:01,478 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:55:01,703 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:55:16,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:55:16,806 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:55:31,657 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:55:32,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:55:32,255 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:55:47,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:55:47,566 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:56:01,731 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:56:02,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:56:02,970 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:56:18,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:56:18,430 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:56:31,806 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:56:33,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:56:33,877 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:56:49,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:56:49,143 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:57:01,881 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:57:04,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:57:04,670 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:57:19,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:57:19,980 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:57:31,958 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:57:35,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:57:35,403 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:57:50,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:57:50,794 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:58:02,033 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:58:06,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:58:06,168 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:58:21,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:58:21,454 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:58:32,111 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:58:36,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:58:36,847 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:58:52,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:58:52,144 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:59:02,187 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:59:07,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:59:07,536 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:59:22,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:59:22,917 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:59:32,261 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 09:59:38,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:59:38,320 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 09:59:53,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 09:59:53,580 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:00:02,337 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:00:08,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:00:08,911 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:00:24,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:00:24,289 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:00:32,417 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:00:39,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:00:39,587 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:00:54,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:00:54,864 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:01:02,493 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:01:10,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:01:10,346 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:01:25,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:01:25,747 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:01:32,571 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:01:41,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:01:41,108 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:01:56,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:01:56,418 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:02:02,646 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:02:11,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:02:11,797 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:02:27,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:02:27,087 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:02:32,721 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:02:42,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:02:42,507 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:02:57,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:02:57,917 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:03:02,799 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:03:13,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:03:13,365 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:03:28,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:03:28,823 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:03:32,878 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:03:44,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:03:44,219 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:03:59,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:03:59,628 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:04:02,953 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:04:15,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:04:15,007 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:04:30,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:04:30,419 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:04:33,033 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:04:45,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:04:45,852 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:05:01,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:05:01,325 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:05:03,111 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:05:16,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:05:16,745 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:05:32,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:05:32,105 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:05:33,187 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:05:47,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:05:47,520 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:06:03,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:06:03,001 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:06:03,265 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:06:18,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:06:18,367 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:06:33,340 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:06:33,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:06:33,821 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:06:49,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:06:49,374 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:07:03,418 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:07:04,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:07:04,652 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:07:20,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:07:20,038 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:07:33,495 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:07:35,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:07:35,433 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:07:50,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:07:50,831 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:08:03,572 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:08:06,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:08:06,256 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:08:21,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:08:21,588 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:08:33,648 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:08:36,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:08:36,902 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:08:52,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:08:52,216 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:09:03,727 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:09:07,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:09:07,505 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:09:23,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:09:23,030 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:09:33,805 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:09:38,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:09:38,510 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:09:53,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:09:53,904 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:10:03,883 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:10:09,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:10:09,140 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:10:24,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:10:24,645 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:10:33,962 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:10:40,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:10:40,155 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:10:55,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:10:55,497 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:11:04,043 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:11:10,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:11:10,863 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:11:26,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:11:26,131 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:11:34,121 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:11:41,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:11:41,492 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:11:56,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:11:56,829 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:12:04,198 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:12:12,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:12:12,144 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:12:27,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:12:27,546 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:12:34,274 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:12:42,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:12:42,892 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:12:58,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:12:58,260 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:13:04,352 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:13:13,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:13:13,593 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:13:28,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:13:28,960 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:13:34,431 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:13:44,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:13:44,286 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:13:51,477 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:13:59,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:13:59,520 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:14:03,482 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:14:04,510 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:14:05,483 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:14:07,484 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:14:09,485 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:14:11,486 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:14:13,486 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:14:14,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:14:14,750 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:14:15,488 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:14:17,489 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:14:19,489 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:14:21,490 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:14:23,491 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:14:25,492 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:14:27,493 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:14:29,493 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:14:30,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:14:30,006 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:14:31,494 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:14:33,495 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:14:34,587 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:14:39,497 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:14:41,498 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:14:45,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:14:45,231 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:14:45,500 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:14:49,501 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:14:53,503 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:14:55,504 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:14:59,506 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:15:00,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:15:00,457 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:15:03,507 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:15:04,666 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:15:05,508 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:15:09,510 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:15:11,511 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:15:13,512 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:15:15,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:15:15,688 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:15:17,513 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:15:19,514 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:15:21,515 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:15:23,516 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:15:27,517 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:15:29,518 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:15:30,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:15:30,915 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:15:31,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:31,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:31,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:31,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:31,519 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:15:31,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:31,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:31,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:31,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:31,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:31,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:31,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:31,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:32,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,520 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:15:33,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:33,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,745 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:15:34,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:34,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,521 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:15:35,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:35,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:36,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,523 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:15:37,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:37,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:38,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,524 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:15:39,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:39,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:40,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:40,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:40,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:40,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:40,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:40,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:40,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:40,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:40,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:40,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:40,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:40,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:40,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:40,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:40,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:40,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:42,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:42,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:42,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:42,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:42,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:42,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:42,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:42,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:42,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:42,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:42,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:42,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:42,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:42,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:42,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:42,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:15:42,526 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:15:44,527 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:15:46,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:15:46,282 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:15:46,527 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:15:48,528 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:15:50,529 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:15:52,530 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:15:54,531 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:15:56,532 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:15:58,532 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:16:00,533 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:16:01,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:16:01,567 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:16:02,534 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:16:04,535 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:16:04,825 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:16:06,536 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:16:08,537 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:16:10,538 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:16:12,538 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:16:14,539 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:16:16,540 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:16:16,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:16:16,801 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:16:18,541 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:16:20,541 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:16:26,544 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:16:28,545 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:16:32,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:16:32,061 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:16:32,546 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:16:34,903 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:16:36,548 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:16:40,550 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:16:42,551 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:16:46,552 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:16:47,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:16:47,276 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:16:50,554 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:16:52,555 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:16:56,557 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:16:58,558 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:00,558 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:02,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:17:02,579 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:17:04,560 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:04,981 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:17:06,561 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:08,562 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:12,564 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:14,564 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:16,565 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:17,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:17:17,796 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:17:18,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:18,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:18,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:18,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:18,566 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:18,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:18,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:18,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:18,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:18,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:18,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:18,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:18,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:19,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:19,189 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,194 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,199 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,204 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,209 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,215 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,220 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,230 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,230 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,230 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,235 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,240 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,246 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,251 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,261 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,267 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,276 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,276 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,277 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,277 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,277 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,277 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,277 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,277 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,277 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,277 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,277 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,277 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,277 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,277 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,277 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,277 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,277 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,277 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,278 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,278 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,278 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,278 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,278 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,278 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,278 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,278 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,278 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,278 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,278 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,278 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,278 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,278 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,278 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,278 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,279 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,279 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,279 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,279 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,279 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,279 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,279 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,279 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,279 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,279 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,279 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,279 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,279 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,279 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,280 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,280 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,280 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,280 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,280 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,280 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,280 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,280 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,280 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,280 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,280 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,280 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,280 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,280 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,280 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,280 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,281 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,281 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,281 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,281 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,281 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,281 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,281 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,281 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,281 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,281 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,281 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,281 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,281 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,281 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,281 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,282 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,282 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,282 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,282 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,282 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,282 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,282 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,282 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,282 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,282 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,282 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,282 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,282 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,282 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,282 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,282 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,283 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,283 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,283 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,283 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,283 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,283 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,283 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,283 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,283 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,283 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,283 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,283 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,283 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,283 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,283 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,283 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,284 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,284 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,284 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,284 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,284 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,284 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,284 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,284 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,284 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,284 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,284 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,284 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,284 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,284 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,284 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,285 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,285 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,285 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,285 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,285 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,285 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,285 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,285 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,285 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,285 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,285 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,285 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,285 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,285 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,285 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,285 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,286 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,286 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,286 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,286 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,286 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,286 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,286 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,286 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,286 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,286 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,286 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,286 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,286 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,286 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,286 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,286 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,287 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,287 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,287 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,287 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,287 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,287 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,287 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,287 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,287 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,287 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,287 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,287 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,287 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,287 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,287 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,287 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,288 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,288 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,288 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,288 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,288 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,288 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,288 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,288 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,288 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,288 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,288 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,288 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,288 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,288 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,288 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,289 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,289 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,289 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,289 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,289 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,289 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,289 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,289 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,289 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,289 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,289 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,289 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,289 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,289 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,289 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,289 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,290 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,290 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,290 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,290 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,290 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,290 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,290 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,290 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,290 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,290 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,290 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,290 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,290 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,290 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,290 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,290 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,291 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,291 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,291 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,291 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,291 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,291 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,291 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,291 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,291 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,291 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,291 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,291 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,291 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,291 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,291 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,292 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,292 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,292 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,292 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,292 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,292 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,292 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,292 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,292 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,292 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,292 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,292 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,292 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,292 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,292 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,292 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,292 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,293 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,293 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,293 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,293 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,293 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,293 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,293 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,293 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,293 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,293 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,293 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,293 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,293 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,293 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,293 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,293 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,294 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,294 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,294 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,294 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,294 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,294 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,294 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,294 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,294 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,294 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,294 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,294 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,294 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,294 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,294 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,294 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,295 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,295 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,295 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,295 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,295 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,295 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,295 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,295 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,295 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,295 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,295 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,295 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,295 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,295 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,295 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,296 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,296 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,296 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,296 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,296 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,296 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,296 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,296 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,296 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,296 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,296 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,296 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,296 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,296 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,296 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,296 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,297 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,297 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,297 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,297 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,297 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,297 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,297 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,297 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,297 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,297 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,297 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,297 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,297 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,297 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,298 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,298 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,298 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,298 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,298 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,298 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,298 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,298 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,298 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,298 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,298 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,298 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,298 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,298 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,298 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,298 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,299 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,299 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,299 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,299 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,299 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,299 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,299 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,299 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,299 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,299 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,299 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,299 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,299 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,299 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,299 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,299 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,300 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,300 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,300 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,300 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,300 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,300 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,300 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,300 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,300 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,300 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,300 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,300 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,300 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,300 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,300 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,300 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,301 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,301 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,301 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,301 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,301 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,301 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,301 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,301 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,301 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,301 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,301 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,301 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,301 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,301 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,302 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,302 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,302 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,302 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,302 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,302 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,302 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,302 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,302 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,302 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,302 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,302 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,302 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,302 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,302 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,302 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,302 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,303 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,303 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,303 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,303 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,303 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,303 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,303 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,303 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,303 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,303 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,303 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,303 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,303 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,303 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,303 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,304 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,304 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,304 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,304 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,304 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,304 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,304 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,304 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,304 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,304 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,304 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,304 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,304 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,304 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,304 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,304 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,305 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,305 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,305 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,305 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,305 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,305 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,305 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,305 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,305 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,305 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,305 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,305 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,305 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,305 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,305 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,306 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,306 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,306 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,306 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,306 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,306 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,306 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,306 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,306 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,306 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,306 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,306 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,306 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,306 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,306 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,306 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,307 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,307 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,307 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,307 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,307 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,307 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,307 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,307 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,307 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,307 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,307 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,307 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,307 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,307 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,307 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,307 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,308 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,308 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,308 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,308 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,308 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,308 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,308 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,308 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,308 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,308 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,308 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,308 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,308 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,308 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,308 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,309 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,309 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,309 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,309 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,309 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,309 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,309 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,309 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,309 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,309 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,309 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,309 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,309 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,309 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,309 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,309 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,310 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,310 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,310 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,310 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,310 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,310 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,310 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,310 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,310 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,310 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,310 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,310 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,310 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,310 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,311 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,311 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,311 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,311 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,311 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,311 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,311 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,311 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,311 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,311 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,311 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,311 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,311 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,311 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,311 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,311 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,311 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,312 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,312 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,312 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,312 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,312 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,312 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,312 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,312 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,312 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,312 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,312 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,312 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,312 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,312 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,312 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,313 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,313 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,313 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,313 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,313 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,313 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,313 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,313 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,313 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,313 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,313 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,313 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,313 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,313 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,313 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,313 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,314 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,314 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,314 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,314 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,314 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,314 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,314 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,314 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,314 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,314 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,314 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,314 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,314 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,314 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,314 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,315 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,315 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,315 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,315 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,315 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,315 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,315 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,315 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,315 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,315 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,315 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,315 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,315 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,315 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,315 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,315 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,316 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,316 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,316 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,316 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,316 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,316 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,316 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,316 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,316 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,316 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,316 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,316 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,316 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,316 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,316 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,317 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,317 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,317 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,317 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,317 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,317 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,317 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,317 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,317 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,317 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,317 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,317 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,317 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,317 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,317 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,317 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,318 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,318 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,318 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,318 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,318 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,318 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,318 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,318 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,318 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,318 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,318 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,318 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,318 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,318 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,318 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,318 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,319 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,319 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,319 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,319 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,319 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,319 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,319 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,319 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,319 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,319 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,319 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,319 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,319 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,319 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,319 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,320 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,320 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,320 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,320 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,320 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,320 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,320 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,320 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,320 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,320 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,320 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,320 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,320 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,320 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,320 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,321 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,321 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,321 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,321 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,321 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,321 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,321 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,321 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,321 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,321 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,321 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,321 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,321 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,321 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,321 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,321 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,322 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,322 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,322 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,322 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,322 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,322 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,322 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,322 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,322 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,322 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,322 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,322 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,322 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,322 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,322 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,322 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,323 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,323 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,323 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,323 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,323 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,323 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,323 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,323 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,323 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,323 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,323 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,323 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,323 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,323 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,324 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,324 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,324 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,324 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,324 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,324 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,324 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,324 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,324 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,324 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,324 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,324 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,324 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,324 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,324 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,325 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,325 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,325 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,325 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,325 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,325 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,325 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,325 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,325 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,325 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,325 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,325 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,325 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,325 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,325 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,325 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,326 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,326 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,326 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,326 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,326 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,326 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,326 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,326 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,326 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,326 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,326 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,326 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,326 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,326 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,326 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,326 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,327 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,327 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,327 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,327 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,327 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,327 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,327 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,327 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,327 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,327 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,327 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,327 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,327 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,327 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,327 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,327 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,328 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,328 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,328 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,328 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,328 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,328 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,328 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,328 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,328 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,328 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,328 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,328 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,328 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,328 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,328 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,329 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,329 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,329 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,329 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,329 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,329 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,329 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,329 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,329 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,329 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,329 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,329 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,329 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,329 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,329 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,330 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,330 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,330 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,330 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,330 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,330 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,330 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,330 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,330 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,330 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,330 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,330 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,330 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,330 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,330 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,330 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,330 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,331 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,331 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,331 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,331 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,331 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,331 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,331 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,331 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,331 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,331 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,331 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,331 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,331 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,331 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,331 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,331 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,332 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,332 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,332 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,332 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,332 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,332 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,332 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,332 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,332 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,332 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,332 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,332 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,332 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,332 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,332 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,333 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,333 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,333 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,333 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,333 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,333 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,333 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,333 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,333 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,333 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,333 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,333 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,333 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,333 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,333 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,333 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,334 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,334 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,334 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,334 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,334 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,334 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,334 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,334 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,334 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,334 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,334 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,334 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,334 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,334 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,334 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,335 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,335 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,335 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,335 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,335 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,335 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,335 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,335 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,335 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,335 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,335 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,335 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,335 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,335 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,335 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,335 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,336 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,336 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,336 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,336 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,336 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,336 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,336 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,336 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,336 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,336 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,336 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,336 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,336 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,336 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,336 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,336 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,337 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,337 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,337 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,337 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,337 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,337 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,337 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,337 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,337 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,337 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,337 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,337 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,337 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,337 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,337 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,338 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,338 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,338 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,338 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,338 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,338 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,338 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,338 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,338 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,338 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,338 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,338 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,338 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,338 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,338 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,338 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,339 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,339 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,339 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,339 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,339 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,339 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,339 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,339 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,339 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,339 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,339 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,339 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,339 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,339 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,339 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,339 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,340 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,340 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,340 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,340 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,340 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,340 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,340 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,340 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,340 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,340 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,340 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,340 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,340 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,340 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,340 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,340 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,341 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,341 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,341 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,341 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,341 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,341 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,341 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,341 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,341 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,341 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,341 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,341 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,341 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,341 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,341 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,341 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,342 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,342 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,342 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,342 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,342 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,342 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,342 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,342 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,342 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,342 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,342 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,342 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,342 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,342 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,342 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,342 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,343 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,343 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,343 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,343 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,343 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,343 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,343 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,343 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,343 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,343 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,343 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,343 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,343 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,343 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,343 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,343 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,344 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,344 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,344 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,344 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,344 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,344 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,344 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,344 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,344 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,344 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,344 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,344 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,344 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,344 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,344 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,345 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,345 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,345 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,345 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,345 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,345 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,345 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,345 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,346 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,346 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,346 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,346 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,346 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,346 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,346 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,346 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,347 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,347 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,347 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,347 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,347 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,347 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,347 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,347 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,347 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,347 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,347 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,347 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,347 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,347 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,347 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,347 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,348 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,348 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,348 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,348 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,348 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,348 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,348 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,348 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,348 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,348 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,348 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,348 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,348 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,348 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,348 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,349 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,349 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,349 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,349 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,349 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,349 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,349 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,349 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,349 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,349 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,349 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,349 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,349 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,349 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,349 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,350 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,350 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,350 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,350 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,350 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,350 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,350 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,350 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,350 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,350 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,350 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,350 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,350 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,350 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,350 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,350 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,351 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,351 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,351 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,351 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,351 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,351 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,351 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,351 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,351 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,351 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,351 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,351 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,351 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,351 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,351 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,351 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,352 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,352 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,352 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,352 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,352 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,352 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,352 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,352 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,352 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,352 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,352 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,352 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,352 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,352 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,353 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,353 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,353 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,353 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,353 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,353 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,353 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,353 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,353 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,353 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,353 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,353 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,353 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,353 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,353 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,353 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,354 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,354 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,354 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,354 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,354 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,354 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,354 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,354 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,354 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,354 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,354 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,354 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,354 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,354 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,354 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,355 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,355 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,355 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,355 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,355 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,355 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,355 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,355 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,355 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,355 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,355 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,355 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,355 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,355 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,355 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,355 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,356 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,356 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,356 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,356 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,356 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,356 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,356 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,356 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,356 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,356 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,356 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,356 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,356 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,356 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,356 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,356 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,357 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,357 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,357 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,357 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,357 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,357 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,357 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,357 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,357 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,357 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,357 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,357 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,357 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,357 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,357 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,357 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,358 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,358 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,358 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,358 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,358 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,358 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,358 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,358 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,358 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,358 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,358 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,358 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,358 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,358 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,358 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,358 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,359 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,359 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,359 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,359 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,359 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,359 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,359 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,359 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,359 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,359 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,359 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,359 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,359 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,359 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,359 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,360 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,360 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,360 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,360 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,360 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,360 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,360 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,360 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,360 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,360 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,360 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,360 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,360 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,360 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,360 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,360 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,361 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,361 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,361 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,361 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,361 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,361 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,361 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,361 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,361 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,361 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,361 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,361 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,361 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,361 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,361 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,361 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,362 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,362 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,362 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,362 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,362 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,362 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,362 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,362 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,362 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,362 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,362 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,362 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,362 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,362 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,362 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,363 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,363 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,363 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,363 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,363 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,363 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,363 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,363 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,363 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,363 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,363 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,363 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,363 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,363 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,363 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,363 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,364 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,364 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,364 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,364 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,364 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,364 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,364 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,364 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,364 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,364 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,364 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,364 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,364 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,364 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,365 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,365 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,365 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,365 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,365 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,365 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,365 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,365 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,365 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,365 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,365 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,365 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,365 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,365 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,365 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,366 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,366 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,366 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,366 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,366 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,366 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,366 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,366 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,366 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,366 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,366 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,366 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,366 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,366 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,366 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,366 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,367 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,367 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,367 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,367 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,367 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,367 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,367 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,367 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,367 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,367 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,367 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,367 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,367 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,367 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,367 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,367 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,368 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,368 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,368 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,368 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,368 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,368 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,368 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,368 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,368 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,368 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,368 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,368 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,368 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,368 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,368 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,369 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,369 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,369 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,369 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,369 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,369 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,369 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,369 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,369 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,369 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,369 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,369 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,369 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,369 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,370 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,370 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,370 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,370 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,370 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,370 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,370 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,370 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,370 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,370 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,370 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,370 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,370 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,370 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,370 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,370 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,370 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,371 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,371 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,371 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,371 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,371 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,371 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,371 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,371 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,371 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,371 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,371 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,371 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,371 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,371 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,371 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,372 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,372 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,372 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,372 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,372 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,372 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,372 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,372 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,372 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,372 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,372 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,372 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,372 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,372 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,373 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,373 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,373 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,373 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,373 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,373 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,373 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,373 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,373 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,373 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,373 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,373 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,373 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,373 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,373 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,373 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,374 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,374 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,374 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,374 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,374 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,374 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,374 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,374 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,374 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,374 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,374 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,374 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,374 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,374 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,374 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,375 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,375 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,375 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,375 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,375 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,375 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,375 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,375 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,375 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,375 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,375 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,375 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,375 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,375 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,375 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,376 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,376 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,376 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,376 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,376 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,376 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,376 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,376 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,376 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,376 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,376 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,376 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,376 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,376 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,376 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,376 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,377 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,377 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,377 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,377 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,377 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,377 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,377 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,377 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,377 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,377 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,377 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,377 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,377 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,377 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,377 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,377 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,378 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,378 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,378 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,378 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,378 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,378 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,378 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,378 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,378 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,378 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,378 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,378 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,378 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,378 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,378 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,378 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,379 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,379 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,379 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,379 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,379 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,379 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,379 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,379 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,379 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,379 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,379 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,379 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,379 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,379 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,379 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,379 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,380 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,380 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,380 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,380 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,380 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,380 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,380 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,380 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,380 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,380 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,380 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,380 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,380 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,380 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,380 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,380 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,381 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,381 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,381 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,381 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,381 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,381 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,381 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,381 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,381 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,381 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,381 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,381 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,381 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,381 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,381 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,381 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,382 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,382 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,382 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,382 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,382 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,382 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,382 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,382 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,382 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,382 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,382 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,382 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,382 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,382 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,382 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,382 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,383 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,383 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,383 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,383 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,383 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,383 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,383 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,383 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,383 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,383 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,383 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,383 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,383 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,383 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,383 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,383 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,384 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,384 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,384 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,384 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,384 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,384 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,384 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,384 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,384 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,384 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,384 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,384 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,384 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,384 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,384 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,384 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,385 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,385 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,385 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,385 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,385 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,385 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,385 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,385 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,385 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,385 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,385 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,385 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,385 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,385 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,385 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,385 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,385 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,386 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,386 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,386 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,386 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,386 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,386 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,386 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,386 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,386 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,386 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,386 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,386 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,386 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,386 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,386 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,387 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,387 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,387 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,387 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,387 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,387 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,387 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,387 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,387 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,387 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,387 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,387 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,387 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,387 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,387 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,387 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,387 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,388 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,388 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,388 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,388 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,388 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,388 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,388 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,388 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,388 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,388 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,388 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,388 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,388 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,388 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,388 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,388 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,389 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,389 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,389 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,389 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,389 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,389 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,389 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,389 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,389 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,389 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,389 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,389 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,389 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,389 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,389 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,389 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,390 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,390 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,390 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,390 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,390 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,390 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,390 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,390 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,390 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,390 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,390 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,390 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,390 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,390 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,390 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,391 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,391 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,391 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,391 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,391 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,391 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,391 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,391 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,391 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,391 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,391 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,391 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,391 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,391 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,391 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,391 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,392 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,392 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,392 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,392 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,392 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,392 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,392 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,392 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,392 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,392 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,392 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,392 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,392 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,392 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,392 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,392 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,393 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,393 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,393 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,393 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,393 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,393 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,393 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,393 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,393 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,393 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,393 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,393 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,393 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,393 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,393 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,393 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,394 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,394 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,394 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,394 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,394 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,394 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,394 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,394 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,394 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,394 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,394 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,394 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,394 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,394 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,394 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,395 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,395 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,395 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,395 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,395 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,395 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,395 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,395 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,395 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,395 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,395 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,395 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,395 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,395 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,395 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,395 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,396 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,396 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,396 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,396 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,396 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,396 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,396 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,396 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,396 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,396 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,396 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,396 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,396 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,396 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,396 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,397 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,397 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,397 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,397 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,397 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,397 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,397 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,397 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,397 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,397 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,397 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,397 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,397 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,397 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,397 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,397 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,398 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,398 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,398 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,398 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,398 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,398 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,398 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,398 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,398 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,398 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,398 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,398 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,398 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,398 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,398 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,398 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,399 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,399 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,399 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,399 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,399 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,399 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,399 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,399 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,399 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,399 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,399 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,399 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,399 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,399 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,399 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,399 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,400 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,400 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,400 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,400 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,400 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,400 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,400 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,400 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,400 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,400 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,400 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,400 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,400 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,400 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,400 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,400 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,401 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,401 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,401 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,401 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,401 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,401 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,401 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,401 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,401 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,401 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,401 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,401 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,401 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,401 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,401 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,401 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,402 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,402 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,402 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,402 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,402 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,402 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,402 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,402 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,402 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,402 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,402 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,402 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,402 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,402 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,402 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,402 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,403 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,403 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,403 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,403 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,403 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,403 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,403 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,403 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,403 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,403 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,403 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,403 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,403 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,403 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,403 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,404 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,404 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,404 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,404 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,404 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,404 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,404 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,404 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,404 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,404 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,404 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,404 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,404 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,404 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,404 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,405 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,405 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,405 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,405 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,405 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,405 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,405 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,405 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,405 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,405 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,405 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,405 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,405 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,405 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,405 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,405 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,406 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,406 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,406 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,406 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,406 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,406 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,406 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,406 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,406 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,406 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,406 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,406 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,406 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,406 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,406 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,406 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,407 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,407 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,407 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,407 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,407 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,407 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,407 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,407 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,407 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,407 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,407 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,407 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,407 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,407 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,407 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,407 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,408 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,408 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,408 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,408 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,408 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,408 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,408 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,408 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,408 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,408 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,408 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,408 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,408 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,408 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,408 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,408 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,408 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,409 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,409 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,409 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,409 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,409 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,409 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,409 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,409 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,409 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,409 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,409 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,409 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,409 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,409 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,409 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,410 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,410 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,410 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,410 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,410 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,410 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,410 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,410 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,410 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,410 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,410 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,410 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,410 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,410 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,410 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,410 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,411 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,411 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,411 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,411 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,411 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,411 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,411 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,411 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,411 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,411 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,411 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,411 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,411 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,411 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,411 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,411 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,412 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,412 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,412 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,412 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,412 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,412 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,412 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,412 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,412 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,412 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,412 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,412 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,412 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,412 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,413 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,413 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,413 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,413 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,413 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,413 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,413 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,413 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,413 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,413 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,413 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,413 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,413 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,413 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,413 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,413 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,414 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,414 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,414 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,414 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,414 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,414 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,414 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,414 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,414 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,414 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,414 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,414 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,414 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,414 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,414 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,414 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,415 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,415 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,415 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,415 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,415 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,415 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,415 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,415 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,415 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,415 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,415 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,415 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,415 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,415 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,415 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,416 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,416 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,416 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,416 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,416 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,416 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,416 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,416 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,416 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,416 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,416 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,416 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,416 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,416 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,416 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,416 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,417 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,417 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,417 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,417 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,417 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,417 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,417 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,417 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,417 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,417 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,417 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,417 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,417 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,417 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,417 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,417 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,418 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,418 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,418 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,418 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,418 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,418 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,418 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,418 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,418 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,418 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,418 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,418 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,418 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,418 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,418 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,418 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,419 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,419 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,419 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,419 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,419 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,419 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,419 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,419 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,419 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,419 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,419 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,419 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,419 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,419 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,420 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,420 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,420 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,420 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,420 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,420 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,420 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,420 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,420 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,420 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,420 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,420 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,420 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,420 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,420 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,420 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,421 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,421 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,421 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,421 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,421 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,421 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,421 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,421 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,421 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,421 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,421 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,421 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,421 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,421 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,421 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,421 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,422 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,422 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,422 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,422 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,422 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,422 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,422 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,422 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,422 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,422 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,422 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,422 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,422 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,422 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,422 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,422 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,423 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,423 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,423 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,423 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,423 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,423 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,423 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,423 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,423 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,423 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,423 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,423 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,423 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,423 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,423 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,423 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,424 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,424 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,424 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,424 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,424 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,424 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,424 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,424 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,424 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,424 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,424 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,424 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,424 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,424 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,424 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,424 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,425 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,425 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,425 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,425 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,425 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,425 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,425 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,425 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,425 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,425 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,425 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,425 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,425 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,425 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,425 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,425 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,425 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,426 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,426 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,426 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,426 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,426 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,426 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,426 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,426 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 10:17:19,426 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 10:17:19,514 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 10:17:19,570 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 10:17:19,570 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 10:17:20,570 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:21,570 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:22,571 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:23,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:23,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,572 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:24,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:24,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:25,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,573 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:26,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:26,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:27,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,641 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/config.yaml -2022-10-25 10:17:28,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:28,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:29,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:30,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:30,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:30,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:30,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:30,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:30,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:30,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:30,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:30,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:30,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:30,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:30,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:30,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:30,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:30,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:30,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:30,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:30,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:30,642 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:31,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:31,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:32,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:33,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:17:33,093 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:17:33,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:33,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:33,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:33,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:33,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:33,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:33,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:33,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:33,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:33,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:33,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:33,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:33,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:33,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:33,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:33,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:33,644 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:34,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:34,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:34,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:34,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:34,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:34,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:34,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:34,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:34,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:34,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:34,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:34,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:34,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:34,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:34,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:34,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:17:35,123 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:17:35,645 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:37,646 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:39,646 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:41,647 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:43,648 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:45,649 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:47,650 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:48,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:17:48,343 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:17:49,651 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:51,652 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:53,653 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:55,654 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:57,655 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:17:59,656 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:18:01,656 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:18:03,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:18:03,569 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:18:03,657 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:18:05,201 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:18:05,658 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:18:07,659 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:18:13,661 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:18:17,663 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:18:18,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:18:18,790 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:18:21,664 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:18:25,666 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:18:29,668 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:18:31,669 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:18:34,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:18:34,114 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:18:35,280 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:18:35,671 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:18:39,672 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:18:41,673 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:18:43,674 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:18:47,676 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:18:49,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:18:49,347 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:18:49,677 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:18:53,679 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:18:55,680 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:18:57,680 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:18:59,681 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:03,683 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:04,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:19:04,572 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:19:05,359 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:19:05,684 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:07,685 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:07,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:07,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:07,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:07,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:08,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:08,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:08,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:08,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:08,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:08,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:08,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:08,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:09,686 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:11,687 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:13,687 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:13,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:13,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:13,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:13,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:13,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:13,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:13,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:13,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:13,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:13,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:13,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:13,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:13,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:13,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:13,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:13,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,688 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:15,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:15,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:16,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:17,689 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:17,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:17,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:17,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:17,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:17,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:17,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:17,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:17,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:17,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:17,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:17,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:17,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:17,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:17,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:17,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:17,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:18,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:19:19,802 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:19:19,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:19,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:20,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:21,691 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:22,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:22,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,691 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:23,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:23,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:24,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,692 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:25,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:25,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:26,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,693 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:27,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:27,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:19:29,694 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:32,696 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:34,696 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:35,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:19:35,086 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:19:35,438 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:19:36,697 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:38,698 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:40,699 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:42,700 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:44,700 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:46,701 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:48,702 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:50,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:19:50,312 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:19:50,703 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:52,703 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:54,704 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:19:56,705 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:20:00,707 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:20:05,516 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:20:05,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:20:05,530 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:20:06,717 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:20:10,719 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:20:14,721 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:20:16,722 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:20:20,724 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:20:20,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:20:20,754 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:20:22,725 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:20:26,726 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:20:28,727 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:20:32,729 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:20:34,730 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:20:35,597 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:20:36,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:20:36,034 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:20:36,731 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:20:40,732 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:20:42,733 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:20:44,734 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:20:46,735 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:20:50,737 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:20:51,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:20:51,298 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:20:52,738 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:20:54,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:20:54,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:20:54,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:20:54,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:20:54,739 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:20:55,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:20:55,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:20:55,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:20:55,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:20:55,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:20:55,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:20:55,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:20:55,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:20:55,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:20:55,376 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 10:20:55,571 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 10:20:55,655 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 10:20:55,739 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 10:20:56,740 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:20:58,740 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:21:00,741 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:21:02,742 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:21:04,743 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:21:05,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:05,687 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:21:06,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:06,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:06,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:06,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:06,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:06,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:06,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:06,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:06,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:06,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:06,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:06,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:06,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:06,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:06,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:06,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:06,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:21:06,519 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:21:06,746 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:21:07,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:07,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:08,747 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:21:09,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:09,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:10,748 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:21:11,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:11,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,748 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:21:12,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:12,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:13,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:14,749 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:21:15,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:15,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:16,751 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:21:17,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:17,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:17,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:17,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:17,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:17,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:17,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:17,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:17,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:17,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:17,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:17,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:17,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:17,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:17,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:17,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:18,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:18,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:18,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:18,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:18,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:18,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:18,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:18,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:18,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:18,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:18,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:18,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:18,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:18,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:18,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:18,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,752 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:21:19,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:19,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:21:21,753 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:21:21,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:21:21,862 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:21:23,754 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:21:25,754 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:21:27,755 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:21:29,756 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:21:31,757 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:21:33,758 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:21:35,759 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:21:35,766 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:21:37,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:21:37,116 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:21:37,760 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:21:39,761 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:21:41,761 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:21:47,764 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:21:51,765 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:21:52,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:21:52,338 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:21:55,767 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:21:59,769 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:22:01,770 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:22:05,772 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:22:05,844 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:22:07,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:22:07,569 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:22:07,772 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:22:11,774 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:22:13,775 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:22:17,777 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:22:19,777 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:22:21,778 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:22:22,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:22:22,803 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:22:25,780 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:22:27,781 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:22:29,782 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:22:33,783 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:22:35,784 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:22:35,922 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:22:37,785 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:22:38,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:22:38,024 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:22:39,786 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:22:40,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:40,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:40,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:40,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:40,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:40,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:40,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:40,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:40,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:40,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:40,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:40,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:41,787 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:22:43,788 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:22:45,789 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:22:47,789 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:22:49,790 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:22:51,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,791 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:22:51,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:51,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:53,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:22:53,314 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:22:53,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:53,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:53,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:53,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:53,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:53,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:53,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:53,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:53,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:53,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:53,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:53,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:53,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:53,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:53,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:53,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:53,792 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:22:55,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:55,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:55,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:55,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:55,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:55,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:55,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:55,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:55,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:55,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:55,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:55,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:55,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:55,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:55,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:55,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:55,793 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:22:56,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:56,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:56,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:56,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:56,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:56,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:56,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:56,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:56,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:56,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:56,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:56,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:56,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:56,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:56,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:56,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:57,794 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:22:58,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:58,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:22:59,794 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:23:00,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:00,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,795 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:23:01,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:01,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:02,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:03,796 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:23:04,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:04,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:05,797 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:23:06,001 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:23:06,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:06,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:06,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:06,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:06,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:06,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:06,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:06,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:06,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:06,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:06,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:06,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:06,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:06,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:06,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:06,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:07,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:07,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:07,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:07,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:07,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:07,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:07,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:07,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:07,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:07,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:07,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:07,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:07,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:07,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:07,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:07,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:07,798 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:23:08,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:23:08,538 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:23:09,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:09,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:09,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:09,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:09,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:09,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:09,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:09,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:09,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:09,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:09,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:09,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:09,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:09,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:09,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:09,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:09,798 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:23:10,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:10,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:10,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:10,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:10,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:10,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:10,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:10,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:10,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:10,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:10,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:10,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:10,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:10,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:10,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:10,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:23:11,799 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:23:13,800 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:23:15,801 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:23:17,802 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:23:19,803 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:23:21,804 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:23:23,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:23:23,763 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:23:23,804 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:23:25,805 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:23:27,806 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:23:32,808 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:23:36,078 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:23:36,810 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:23:39,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:23:39,002 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:23:40,812 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:23:44,814 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:23:48,815 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:23:50,816 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:23:54,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:23:54,235 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:23:54,818 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:23:58,820 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:00,821 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:04,822 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:06,157 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:24:06,823 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:09,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:24:09,537 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:24:10,825 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:12,826 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:14,827 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:16,827 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:20,829 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:22,830 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:24,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:24:24,762 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:24:24,831 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:26,832 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:27,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:27,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:27,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:27,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:27,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:27,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:27,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:27,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:27,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:27,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:27,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:27,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:27,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:27,858 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 10:24:28,081 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 10:24:28,160 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 10:24:28,932 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:28,933 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 10:24:32,934 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:34,935 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:36,283 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:24:36,936 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:38,936 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:39,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:24:39,987 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:24:40,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:40,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:40,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:40,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:40,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:40,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:40,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:40,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:40,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:40,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:40,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:40,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:40,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:40,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:40,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:40,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:40,937 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:42,938 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:43,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:43,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:43,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:43,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:43,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:43,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:43,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:43,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:43,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:43,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:43,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:43,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:43,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:43,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:43,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:43,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:44,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:44,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:44,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:44,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:44,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:44,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:44,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:44,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:44,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:44,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:44,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:44,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:44,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:44,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:44,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:44,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:44,939 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:46,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:46,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:46,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:46,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:46,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:46,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:46,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:46,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:46,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:46,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:46,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:46,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:46,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:46,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:46,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:46,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:46,940 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:48,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,941 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:48,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:48,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:49,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:50,942 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:51,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:51,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:52,942 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:53,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:53,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,943 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:54,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:54,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:24:55,217 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:24:55,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:55,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:56,944 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:24:58,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:24:58,945 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:25:00,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:25:00,946 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:25:02,947 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:25:04,948 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:25:06,364 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:25:06,949 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:25:08,949 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:25:10,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:25:10,466 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:25:10,950 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:25:12,951 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:25:14,952 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:25:16,952 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:25:20,954 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:25:24,956 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:25:25,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:25:25,796 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:25:28,958 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:25:32,960 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:25:36,443 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:25:36,961 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:25:38,962 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:25:41,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:25:41,034 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:25:42,964 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:25:46,966 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:25:48,966 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:25:50,967 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:25:54,969 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:25:56,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:25:56,357 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:25:56,970 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:25:58,971 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:02,972 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:04,973 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:06,523 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:26:06,974 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:08,975 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:10,976 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:11,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:26:11,606 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:26:12,976 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:14,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:14,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:14,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:14,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:14,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:14,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:14,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:14,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:14,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:14,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:14,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:14,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:16,978 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:18,979 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:20,980 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:23,981 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:25,982 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:26,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:26:26,836 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:26:27,983 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:28,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:28,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:28,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:28,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:28,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:28,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:28,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:28,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:28,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:28,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:28,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:28,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:28,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:28,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:28,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:28,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:29,984 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:31,985 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:33,985 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:34,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:34,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:34,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:34,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:34,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:34,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:34,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:34,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:34,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:34,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:34,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:34,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:34,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:34,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:34,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:34,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:35,986 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:36,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:36,602 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:26:37,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:37,987 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:38,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:38,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:39,988 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:40,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:40,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:41,989 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:42,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:26:42,058 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:26:42,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:42,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:43,990 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:44,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:44,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:45,991 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:46,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:46,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:47,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:47,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:47,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:47,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:47,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:47,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:47,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:47,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:47,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:47,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:47,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:47,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:47,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:47,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:47,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:47,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:47,991 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:48,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:48,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:48,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:48,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:48,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:48,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:48,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:48,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:48,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:48,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:48,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:48,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:48,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:48,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:48,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:48,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:49,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:49,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:49,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:49,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:49,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:49,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:49,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:49,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:49,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:49,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:49,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:49,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:49,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:49,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:49,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:49,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:49,992 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:50,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:50,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:50,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:50,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:50,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:50,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:50,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:50,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:50,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:50,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:50,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:50,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:50,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:50,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:50,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:50,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:26:51,993 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:53,994 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:55,994 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:57,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:26:57,302 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:26:57,995 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:26:59,996 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:27:01,997 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:27:06,683 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:27:07,999 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:27:12,001 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:27:12,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:27:12,557 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:27:16,003 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:27:20,004 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:27:24,006 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:27:26,007 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:27:27,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:27:27,780 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:27:30,009 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:27:34,011 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:27:36,012 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:27:36,763 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:27:38,013 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:27:42,015 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:27:43,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:27:43,177 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:27:44,016 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:27:48,017 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:27:50,018 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:27:52,019 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:27:54,020 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:27:56,021 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:27:58,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:27:58,422 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:28:00,022 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:01,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:01,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:01,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:01,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:02,023 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:02,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:02,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:02,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:02,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:02,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:02,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:02,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:02,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:02,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:02,483 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 10:28:02,717 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 10:28:02,788 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 10:28:03,024 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 10:28:04,024 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:06,025 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:06,900 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:28:08,026 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:10,027 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:13,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:28:13,646 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:28:14,028 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:16,029 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:18,030 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:20,031 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:22,031 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:23,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:23,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:23,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:23,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:23,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:23,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:23,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:23,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:23,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:23,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:23,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:23,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:23,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:23,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:23,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:23,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:24,032 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:25,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:25,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:25,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:25,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:25,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:25,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:25,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:25,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:25,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:25,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:25,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:25,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:25,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:25,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:25,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:25,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,033 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:26,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:26,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:27,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:27,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:27,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:27,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:27,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:27,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:27,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:27,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:27,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:27,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:27,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:27,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:27,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:27,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:27,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:27,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:28,034 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:28,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:28:28,870 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:28:29,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:29,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,035 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:30,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,448 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:30,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:31,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:31,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:31,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:31,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:31,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:31,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:31,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:31,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:31,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:31,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:31,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:31,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:31,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:31,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:31,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:31,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,036 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:32,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:32,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:33,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,036 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:34,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:34,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:35,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,037 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:36,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,448 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,448 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:36,981 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:28:37,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:37,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:38,038 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:38,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:38,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:38,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:38,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:38,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:38,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:38,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:38,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:38,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:38,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:38,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:38,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:38,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:38,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:38,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:38,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:39,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,039 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:40,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:40,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:42,040 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:42,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:42,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:42,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:42,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:42,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:42,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:42,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:42,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:42,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:42,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:42,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:42,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:42,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:42,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:42,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:42,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:28:44,040 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:44,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:28:44,092 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:28:46,041 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:48,042 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:50,043 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:56,045 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:28:59,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:28:59,324 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:29:00,047 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:29:04,049 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:29:06,050 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:29:07,060 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:29:10,051 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:29:14,053 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:29:14,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:29:14,545 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:29:16,054 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:29:21,056 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:29:23,057 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:29:27,059 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:29:29,059 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:29:29,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:29:29,770 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:29:33,061 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:29:35,062 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:29:37,063 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:29:37,139 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:29:39,064 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:29:43,065 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:29:44,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:29:44,994 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:29:45,066 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:29:47,067 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:29:48,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:29:48,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:29:48,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:29:48,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:29:49,068 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:29:49,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:29:49,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:29:49,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:29:49,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:29:49,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:29:49,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:29:49,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:29:49,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:29:51,069 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:29:53,070 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:29:55,070 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:29:57,071 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:29:59,072 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:30:00,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:30:00,251 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:30:01,073 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:30:03,074 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:30:06,075 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:30:07,217 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:30:08,076 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:30:10,077 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:30:12,078 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:30:13,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:13,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:13,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:13,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:13,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:13,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:13,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:13,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:13,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:13,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:13,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:13,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:13,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:13,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:13,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:13,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:14,079 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:30:15,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:30:15,494 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:30:16,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,080 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:30:16,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:16,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:17,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:17,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:17,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:17,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:17,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:17,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:17,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:17,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:17,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:17,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:17,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:17,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:17,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:17,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:17,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:17,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:18,081 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:30:19,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:19,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,082 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:30:20,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:20,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:21,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:21,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:21,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:21,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:21,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:21,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:21,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:21,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:21,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:21,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:21,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:21,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:21,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:21,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:21,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:21,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,082 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:30:22,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:22,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:23,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,083 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:30:24,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:24,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:25,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,084 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:30:26,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:26,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,085 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:30:28,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:28,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:29,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:29,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:29,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:29,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:29,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:29,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:29,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:29,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:29,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:29,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:29,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:29,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:29,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:29,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:29,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:29,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,086 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:30:30,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:30,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:30:30,746 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:30:31,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:31,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:32,087 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:30:32,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:32,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:32,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:32,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:32,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:32,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:32,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:32,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:32,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:32,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:32,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:32,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:32,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:32,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:32,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:32,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:33,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:33,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:33,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:33,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:33,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:33,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:33,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:33,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:33,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:33,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:33,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:33,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:33,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:33,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:33,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:33,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:30:34,087 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:30:36,088 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:30:37,297 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:30:38,089 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:30:44,092 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:30:46,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:30:46,117 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:30:48,094 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:30:52,096 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:30:54,096 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:30:58,098 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:01,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:31:01,424 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:31:02,100 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:04,101 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:07,380 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:31:08,103 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:10,104 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:12,105 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:16,106 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:16,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:31:16,750 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:31:18,107 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:22,109 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:24,109 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:26,110 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:28,111 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:30,112 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:31,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:31:31,973 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:31:34,114 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:35,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:31:35,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:31:35,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:31:35,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:31:36,115 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:36,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:31:36,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:31:36,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:31:36,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:31:36,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:31:36,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:31:36,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:31:36,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:31:36,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 10:31:36,659 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 10:31:36,895 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 10:31:36,980 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 10:31:37,115 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 10:31:37,484 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:31:38,116 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:39,116 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:40,116 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:43,118 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:45,119 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:47,120 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:47,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:31:47,273 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:31:49,121 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:51,122 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:53,123 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:55,124 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:57,124 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:31:59,125 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:01,126 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:02,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:32:02,571 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:32:03,127 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:05,128 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:07,129 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:07,562 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:32:09,130 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:11,131 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:13,132 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:15,133 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:17,134 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:17,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:32:17,798 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:32:19,134 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:21,135 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:23,136 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:25,137 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:27,138 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:29,139 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:31,140 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:33,141 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:33,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:32:33,341 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:32:35,141 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:37,142 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:37,641 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:32:39,143 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:41,144 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:43,145 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:45,146 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:47,147 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:48,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:32:48,555 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:32:49,147 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:51,148 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:53,149 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:56,150 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:32:58,151 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:00,152 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:02,153 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:03,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:33:03,811 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:33:04,154 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:06,155 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:07,718 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:33:08,156 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:10,157 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:12,158 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:14,158 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:16,159 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:18,160 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:19,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:33:19,031 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:33:20,161 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:22,161 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:24,162 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:26,163 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:28,164 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:30,165 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:32,166 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:34,166 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:34,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:33:34,393 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:33:36,167 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:37,795 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:33:38,168 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:40,169 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:42,170 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:44,170 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:46,171 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:48,172 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:49,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:33:49,692 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:33:50,173 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:52,174 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:54,175 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:56,176 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:33:58,177 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:00,177 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:02,178 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:04,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:34:04,926 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:34:05,179 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:07,180 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:07,872 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:34:09,181 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:11,182 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:13,183 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:15,184 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:17,185 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:19,186 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:20,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:34:20,249 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:34:21,187 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:23,188 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:25,188 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:27,189 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:29,190 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:31,191 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:33,192 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:35,192 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:35,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:34:35,562 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:34:37,193 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:37,952 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:34:39,194 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:41,195 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:43,196 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:45,197 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:47,198 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:49,199 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:50,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:34:50,914 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:34:51,199 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:53,200 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:55,201 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:57,202 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:34:59,203 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:01,204 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:03,204 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:06,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:35:06,140 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:35:06,206 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:08,031 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:35:08,207 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:10,208 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:12,209 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:14,209 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:16,210 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:18,211 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:20,212 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:21,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:35:21,473 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:35:22,213 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:24,214 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:26,215 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:28,216 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:30,217 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:32,217 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:34,218 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:36,219 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:36,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:35:36,789 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:35:38,109 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:35:38,219 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:40,220 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:42,221 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:44,222 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:46,223 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:48,224 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:50,225 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:52,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:35:52,049 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:35:52,226 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:54,227 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:56,228 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:35:58,229 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:00,230 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:02,231 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:04,232 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:06,233 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:07,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:36:07,352 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:36:08,189 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:36:08,234 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:10,235 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:12,236 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:14,237 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:16,238 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:18,239 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:21,240 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:22,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:36:22,662 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:36:23,241 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:25,242 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:27,243 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:29,245 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:31,246 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:33,247 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:35,248 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:37,249 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:37,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:36:37,887 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:36:38,269 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:36:39,250 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:41,251 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:43,252 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:45,253 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:47,254 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:49,255 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:51,256 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:53,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:36:53,158 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:36:53,256 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:55,258 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:57,258 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:36:59,259 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:01,260 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:03,260 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:05,261 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:07,262 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:08,348 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:37:08,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:37:08,518 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:37:09,263 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:11,264 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:13,265 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:16,266 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:18,267 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:20,268 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:22,269 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:23,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:37:23,742 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:37:24,270 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:26,272 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:28,272 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:30,274 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:32,275 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:34,276 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:36,277 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:38,278 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:38,427 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:37:38,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:37:38,965 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:37:40,279 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:42,280 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:44,281 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:46,282 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:48,283 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:50,284 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:52,285 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:54,286 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:54,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:37:54,397 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:37:56,286 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:37:58,287 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:00,288 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:02,289 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:04,290 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:06,291 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:08,292 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:08,506 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:38:09,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:38:09,746 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:38:10,293 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:12,294 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:14,295 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:16,296 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:18,297 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:20,298 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:22,300 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:24,300 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:25,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:38:25,040 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:38:26,301 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:29,303 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:31,304 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:33,305 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:35,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:37,307 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:38,587 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:38:39,308 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:40,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:38:40,262 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:38:41,309 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:43,310 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:45,311 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:47,312 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:49,313 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:51,314 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:53,315 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:55,316 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:55,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:38:55,485 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:38:57,318 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:38:59,319 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:01,320 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:03,321 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:05,322 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:07,323 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:08,670 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:39:09,324 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:10,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:39:10,717 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:39:11,325 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:13,326 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:15,326 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:17,327 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:19,328 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:21,329 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:23,330 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:25,331 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:26,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:39:26,054 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:39:27,332 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:29,333 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:31,334 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:33,335 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:35,336 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:37,337 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:38,751 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:39:39,337 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:41,338 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:41,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:39:41,433 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:39:43,339 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:45,340 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:47,341 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:49,342 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:51,343 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:53,344 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:55,345 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:39:56,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:39:56,658 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:39:58,346 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:00,347 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:02,348 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:04,349 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:06,350 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:08,351 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:08,828 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:40:10,352 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:12,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:40:12,008 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:40:12,353 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:14,354 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:16,355 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:18,356 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:20,357 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:22,358 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:24,359 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:26,360 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:27,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:40:27,359 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:40:28,365 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:30,365 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:32,366 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:34,367 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:36,368 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:38,369 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:38,908 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:40:40,370 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:42,371 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:42,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:40:42,648 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:40:44,372 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:46,373 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:48,374 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:50,375 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:52,376 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:54,377 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:56,378 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:40:57,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:40:57,944 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:40:58,378 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:00,379 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:02,380 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:04,381 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:06,382 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:08,383 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:08,988 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:41:10,384 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:12,385 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:13,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:41:13,237 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:41:14,386 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:17,387 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:19,388 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:21,389 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:23,390 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:25,391 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:27,392 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:28,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:41:28,467 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:41:29,393 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:31,394 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:33,395 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:35,396 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:37,397 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:39,068 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:41:39,398 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:41,399 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:43,400 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:43,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:41:43,788 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:41:45,401 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:47,402 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:49,402 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:51,403 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:53,404 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:55,405 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:57,406 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:41:59,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:41:59,077 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:41:59,407 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:01,408 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:03,409 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:05,410 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:07,410 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:09,148 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:42:09,411 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:11,412 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:13,413 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:15,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:42:15,067 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:42:15,414 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:17,415 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:19,416 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:21,417 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:23,418 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:25,418 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:28,420 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:30,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:42:30,291 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:42:30,421 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:32,422 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:34,423 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:36,424 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:38,425 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:39,229 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:42:40,426 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:42,426 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:44,427 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:45,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:42:45,516 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:42:46,428 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:48,429 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:50,430 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:52,431 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:54,432 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:56,433 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:42:58,434 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:00,435 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:00,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:43:00,809 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:43:02,436 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:04,437 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:06,438 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:08,439 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:09,309 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:43:10,439 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:12,440 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:14,441 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:16,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:43:16,059 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:43:16,442 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:18,443 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:20,444 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:22,445 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:24,446 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:26,447 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:28,448 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:30,449 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:31,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:43:31,295 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:43:32,450 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:34,451 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:37,452 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:39,390 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:43:39,453 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:41,454 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:43,455 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:45,456 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:46,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:43:46,598 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:43:47,457 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:49,458 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:51,459 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:53,460 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:55,461 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:57,462 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:43:59,463 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:01,464 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:01,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:44:01,965 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:44:03,465 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:05,466 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:07,466 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:09,468 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:09,471 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:44:11,469 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:13,470 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:15,471 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:17,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:44:17,203 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:44:17,472 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:19,473 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:21,474 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:23,475 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:25,476 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:27,476 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:29,477 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:31,478 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:32,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:44:32,450 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:44:33,479 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:35,480 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:37,481 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:39,482 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:39,551 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:44:41,483 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:43,484 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:45,485 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:47,486 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:47,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:44:47,758 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:44:49,487 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:51,488 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:53,489 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:55,489 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:57,490 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:44:59,491 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:01,492 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:02,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:45:02,984 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:45:03,493 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:05,494 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:07,495 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:09,496 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:09,631 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:45:11,497 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:13,498 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:15,499 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:17,500 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:18,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:45:18,355 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:45:19,500 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:21,501 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:23,502 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:25,503 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:27,504 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:29,505 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:31,506 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:33,507 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:33,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:45:33,578 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:45:35,508 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:37,509 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:39,509 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:39,711 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:45:42,511 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:44,512 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:46,513 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:48,514 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:48,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:45:48,872 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:45:50,515 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:52,516 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:54,517 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:56,517 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:45:58,518 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:00,519 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:02,520 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:04,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:46:04,184 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:46:04,521 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:06,522 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:08,523 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:09,791 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:46:10,524 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:12,524 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:14,525 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:16,526 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:18,527 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:19,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:46:19,410 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:46:20,528 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:22,529 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:24,529 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:26,530 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:28,531 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:30,531 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:32,532 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:34,533 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:34,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:46:34,668 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:46:36,534 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:38,534 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:39,871 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:46:40,535 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:42,536 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:44,537 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:46,537 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:48,538 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:50,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:46:50,026 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:46:50,539 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:52,540 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:55,541 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:57,541 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:46:59,542 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:01,543 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:03,544 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:05,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:47:05,341 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:47:05,544 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:07,545 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:09,546 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:09,951 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:47:11,547 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:13,548 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:15,549 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:17,549 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:19,550 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:20,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:47:20,570 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:47:21,551 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:23,552 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:25,553 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:27,553 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:29,554 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:31,555 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:33,556 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:35,557 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:35,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:47:35,796 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:47:37,558 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:39,559 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:40,029 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:47:41,559 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:43,560 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:45,561 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:47,562 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:49,563 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:51,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:47:51,130 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:47:51,563 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:53,564 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:55,565 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:57,565 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:47:59,566 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:01,567 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:03,568 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:05,569 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:06,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:48:06,368 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:48:08,570 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:10,108 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:48:10,571 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:12,572 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:14,573 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:16,573 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:18,574 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:20,575 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:21,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:48:21,592 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:48:22,576 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:24,577 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:26,578 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:28,579 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:30,579 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:32,580 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:34,581 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:36,582 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:36,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:48:36,817 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:48:38,583 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:40,186 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:48:40,584 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:42,584 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:44,585 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:46,586 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:48,587 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:50,588 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:52,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:48:52,040 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:48:52,589 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:54,590 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:56,591 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:48:58,592 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:00,592 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:02,593 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:04,594 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:06,595 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:07,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:49:07,315 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:49:08,596 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:10,266 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:49:10,597 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:12,598 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:14,598 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:16,599 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:18,600 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:20,601 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:22,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:49:22,533 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:49:22,601 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:24,602 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:26,603 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:28,604 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:31,605 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:33,605 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:35,606 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:37,607 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:37,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:49:37,961 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:49:39,608 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:40,345 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:49:41,608 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:43,609 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:45,610 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:47,611 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:49,612 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:51,613 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:53,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:49:53,217 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:49:53,614 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:55,615 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:57,616 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:49:59,616 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:01,617 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:03,618 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:05,619 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:07,620 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:08,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:50:08,445 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:50:09,620 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:10,425 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:50:11,621 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:13,622 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:15,623 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:17,624 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:19,625 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:21,626 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:23,626 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:23,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:50:23,751 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:50:25,627 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:27,628 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:29,629 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:31,630 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:33,631 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:35,631 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:37,632 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:39,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:50:39,073 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:50:39,633 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:40,503 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:50:41,634 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:43,635 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:45,636 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:47,636 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:49,637 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:51,638 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:53,639 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:54,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:50:54,476 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:50:56,640 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:50:58,641 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:00,642 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:02,643 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:04,644 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:06,645 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:08,645 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:09,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:51:09,696 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:51:10,581 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:51:10,646 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:12,647 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:14,648 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:16,649 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:18,650 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:20,651 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:22,652 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:24,653 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:25,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:51:25,006 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:51:26,654 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:28,654 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:30,655 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:32,656 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:34,657 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:36,658 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:38,659 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:40,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:51:40,267 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:51:40,660 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:51:40,664 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:42,665 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:44,665 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:46,666 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:48,667 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:50,668 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:52,669 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:54,670 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:55,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:51:55,563 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:51:56,670 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:51:58,671 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:00,672 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:03,674 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:05,675 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:07,676 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:09,676 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:10,739 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:52:10,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:52:10,790 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:52:11,677 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:13,678 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:15,679 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:17,680 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:19,681 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:21,682 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:23,683 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:25,684 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:26,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:52:26,012 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:52:27,685 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:29,685 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:31,686 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:33,687 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:35,687 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:37,688 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:39,689 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:40,819 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:52:41,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:52:41,417 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:52:41,690 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:43,690 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:45,691 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:47,692 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:49,693 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:51,693 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:53,694 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:55,695 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:56,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:52:56,669 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:52:57,696 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:52:59,696 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:01,697 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:03,698 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:05,699 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:07,699 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:09,700 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:10,900 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:53:11,701 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:11,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:53:11,984 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:53:13,702 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:15,702 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:17,703 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:19,704 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:21,704 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:23,705 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:25,706 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:27,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:53:27,208 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:53:27,707 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:29,707 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:32,708 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:34,709 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:36,710 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:38,710 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:40,711 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:40,979 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:53:42,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:53:42,537 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:53:42,712 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:44,713 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:46,713 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:48,714 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:50,715 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:52,715 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:54,716 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:56,717 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:53:57,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:53:57,761 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:53:58,717 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:00,718 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:02,719 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:04,720 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:06,720 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:08,721 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:10,722 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:11,059 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:54:12,723 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:13,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:54:13,030 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:54:14,723 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:16,724 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:18,725 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:20,725 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:22,726 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:24,727 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:26,728 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:28,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:54:28,257 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:54:28,728 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:30,729 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:32,730 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:34,731 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:36,732 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:38,732 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:40,733 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:41,138 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:54:42,734 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:43,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:54:43,486 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:54:44,735 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:46,735 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:48,736 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:50,737 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:52,738 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:54,739 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:56,740 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:58,740 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:54:58,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:54:58,776 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:55:00,741 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:02,742 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:05,744 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:07,745 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:09,746 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:11,217 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:55:11,746 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:13,747 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:14,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:55:14,086 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:55:15,748 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:17,749 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:19,750 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:21,751 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:23,752 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:25,753 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:27,754 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:29,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:55:29,371 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:55:29,755 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:31,756 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:33,757 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:35,757 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:37,758 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:39,759 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:41,296 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:55:41,760 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:43,760 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:44,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:55:44,595 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:55:45,761 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:47,762 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:49,763 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:51,764 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:53,764 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:55,765 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:57,766 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:59,767 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:55:59,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:55:59,985 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:56:01,768 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:03,769 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:05,770 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:07,771 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:10,772 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:11,375 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:56:12,773 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:14,774 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:15,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:56:15,303 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:56:16,775 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:18,776 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:20,777 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:22,778 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:24,779 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:26,780 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:28,781 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:30,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:56:30,622 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:56:30,782 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:32,783 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:34,784 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:36,785 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:38,786 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:40,787 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:41,457 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:56:42,788 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:44,789 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:45,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:56:45,860 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:56:46,790 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:48,791 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:50,792 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:52,793 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:54,793 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:56,794 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:56:58,795 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:00,796 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:01,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:57:01,265 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:57:02,797 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:04,798 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:06,799 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:08,800 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:11,535 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:57:11,801 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:13,802 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:15,803 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:16,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:57:16,495 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:57:17,804 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:19,805 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:21,806 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:23,807 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:25,808 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:27,809 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:29,810 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:31,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:57:31,722 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:57:31,811 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:33,812 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:35,813 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:37,814 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:39,815 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:41,614 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:57:41,815 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:43,816 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:45,817 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:46,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:57:46,941 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:57:47,818 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:49,819 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:51,820 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:53,821 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:55,822 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:57,823 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:57:59,824 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:01,825 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:02,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:58:02,246 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:58:03,826 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:05,826 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:07,827 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:09,828 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:11,694 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:58:11,829 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:13,829 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:15,830 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:17,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:58:17,507 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:58:17,831 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:20,832 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:22,833 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:24,834 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:26,835 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:28,836 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:30,837 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:32,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:58:32,814 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:58:32,837 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:34,838 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:36,839 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:38,840 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:40,841 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:41,772 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:58:42,842 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:44,842 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:46,843 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:48,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:58:48,037 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:58:48,844 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:50,845 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:52,846 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:54,846 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:56,847 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:58:58,848 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:00,849 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:02,849 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:03,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:59:03,383 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:59:04,850 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:06,851 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:08,852 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:10,853 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:11,851 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:59:12,856 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:14,857 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:16,857 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:18,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:59:18,714 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:59:18,858 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:20,859 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:22,860 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:24,860 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:26,861 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:28,862 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:30,863 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:32,864 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:34,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:59:34,010 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:59:34,864 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:36,865 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:38,866 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:40,867 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:41,929 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 10:59:42,868 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:44,868 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:46,869 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:48,870 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:49,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 10:59:49,282 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 10:59:50,871 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:52,871 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:54,872 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:56,873 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 10:59:58,874 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:01,875 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:03,876 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:04,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:00:04,601 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:00:05,877 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:07,878 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:09,879 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:11,879 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:12,008 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:00:13,880 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:15,881 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:17,882 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:19,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:00:19,826 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:00:19,883 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:21,884 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:23,885 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:25,886 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:27,886 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:29,887 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:31,888 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:33,889 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:35,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:00:35,040 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:00:35,890 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:37,891 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:39,892 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:41,892 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:42,086 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:00:43,893 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:45,894 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:47,895 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:49,896 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:50,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:00:50,335 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:00:51,897 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:53,898 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:55,899 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:00:58,900 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:00,901 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:02,901 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:04,902 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:05,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:01:05,712 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:01:06,903 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:08,904 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:10,905 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:12,167 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:01:12,906 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:14,907 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:16,908 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:18,909 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:20,910 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:20,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:01:20,973 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:01:22,910 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:24,911 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:26,912 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:28,913 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:30,914 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:32,915 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:34,916 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:36,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:01:36,269 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:01:36,917 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:38,917 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:40,918 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:42,246 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:01:42,919 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:44,920 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:46,921 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:48,921 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:50,922 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:51,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:01:51,488 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:01:52,923 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:54,924 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:56,924 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:01:58,925 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:00,926 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:02,926 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:04,927 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:06,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:02:06,743 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:02:06,928 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:08,929 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:10,930 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:12,326 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:02:12,930 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:14,931 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:16,932 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:18,933 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:20,933 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:21,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:02:21,975 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:02:22,934 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:24,935 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:26,936 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:28,936 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:30,937 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:32,938 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:34,939 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:36,940 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:37,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:02:37,338 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:02:39,941 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:41,942 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:42,406 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:02:43,942 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:45,943 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:47,944 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:49,944 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:51,945 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:52,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:02:52,701 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:02:53,946 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:55,947 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:57,947 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:02:59,948 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:01,949 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:03,950 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:05,951 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:07,951 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:07,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:03:07,993 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:03:09,952 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:11,953 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:12,486 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:03:13,954 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:15,954 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:17,955 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:19,956 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:21,957 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:23,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:03:23,214 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:03:23,957 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:25,958 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:27,959 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:29,960 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:31,960 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:33,961 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:35,962 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:37,963 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:38,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:03:38,587 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:03:39,963 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:41,964 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:42,564 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:03:43,965 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:45,966 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:47,966 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:49,967 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:51,968 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:53,969 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:54,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:03:54,011 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:03:55,969 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:57,970 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:03:59,971 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:01,971 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:03,972 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:05,973 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:07,974 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:09,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:04:09,236 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:04:09,974 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:12,644 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:04:12,975 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:14,976 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:16,977 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:18,978 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:20,978 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:22,979 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:24,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:04:24,912 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:04:24,980 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:26,981 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:28,982 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:30,982 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:32,983 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:34,984 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:36,985 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:38,985 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:40,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:04:40,288 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:04:40,986 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:42,723 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:04:42,987 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:44,988 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:46,988 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:48,989 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:50,990 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:52,990 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:54,991 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:55,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:04:55,525 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:04:56,992 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:04:58,993 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:00,994 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:02,994 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:05,995 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:07,996 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:09,997 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:10,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:05:10,744 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:05:11,998 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:12,802 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:05:13,998 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:15,999 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:18,000 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:20,001 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:22,001 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:24,002 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:26,003 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:26,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:05:26,129 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:05:28,004 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:30,004 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:32,005 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:34,006 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:36,007 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:38,008 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:40,008 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:41,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:05:41,354 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:05:42,009 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:42,881 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:05:44,010 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:46,011 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:48,012 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:50,012 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:52,013 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:54,014 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:56,015 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:05:56,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:05:56,752 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:05:58,016 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:00,016 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:02,017 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:04,018 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:06,019 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:08,020 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:10,020 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:11,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:06:11,979 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:06:12,021 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:12,960 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:06:14,022 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:16,023 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:18,024 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:20,025 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:22,026 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:24,027 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:26,027 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:27,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:06:27,364 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:06:28,028 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:31,029 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:33,030 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:35,031 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:37,032 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:39,033 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:41,034 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:42,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:06:42,593 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:06:43,034 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:43,039 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:06:45,035 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:47,036 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:49,037 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:51,038 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:53,039 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:55,040 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:57,040 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:06:57,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:06:57,884 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:06:59,041 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:01,042 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:03,043 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:05,044 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:07,044 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:09,045 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:11,046 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:13,047 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:13,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:07:13,113 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:07:13,339 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:07:15,048 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:17,048 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:19,049 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:21,050 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:23,051 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:25,052 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:27,052 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:28,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:07:28,426 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:07:29,053 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:31,054 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:33,055 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:35,055 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:37,056 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:40,057 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:42,058 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:43,198 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:07:43,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:07:43,730 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:07:44,059 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:46,060 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:48,061 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:50,061 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:52,062 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:54,063 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:56,064 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:58,065 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:07:59,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:07:59,115 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:08:00,066 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:02,066 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:04,067 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:06,068 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:08,069 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:10,070 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:12,070 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:13,277 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:08:14,071 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:15,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:08:15,140 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:08:16,072 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:18,072 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:20,073 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:22,074 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:24,075 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:26,076 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:28,076 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:30,077 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:30,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:08:30,393 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:08:32,078 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:34,079 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:36,080 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:38,081 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:40,081 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:42,082 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:43,357 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:08:44,083 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:45,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:08:45,725 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:08:46,084 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:48,085 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:50,085 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:52,086 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:54,087 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:56,088 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:08:58,089 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:00,090 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:01,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:09:01,025 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:09:02,092 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:05,092 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:07,093 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:09,094 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:11,094 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:13,095 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:13,437 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:09:15,096 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:16,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:09:16,302 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:09:17,097 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:19,097 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:21,098 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:23,099 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:25,099 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:27,100 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:29,100 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:31,101 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:31,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:09:31,523 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:09:33,102 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:35,103 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:37,104 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:39,105 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:41,106 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:43,107 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:43,517 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:09:45,108 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:46,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:09:46,808 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:09:47,109 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:49,110 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:51,111 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:53,112 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:55,113 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:57,114 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:09:59,114 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:01,115 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:02,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:10:02,121 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:10:04,117 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:06,118 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:08,119 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:10,120 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:12,121 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:13,596 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:10:14,122 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:16,123 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:17,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:10:17,600 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:10:18,124 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:20,124 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:22,125 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:24,126 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:26,127 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:28,129 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:30,130 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:32,130 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:32,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:10:32,845 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:10:34,131 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:36,132 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:38,133 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:40,134 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:42,135 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:43,676 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:10:44,136 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:46,137 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:48,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:10:48,072 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:10:48,138 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:50,139 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:52,140 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:54,141 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:56,142 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:10:58,143 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:00,144 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:02,145 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:03,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:11:03,341 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:11:04,146 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:06,147 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:09,148 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:11,149 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:13,149 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:13,755 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:11:15,151 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:17,151 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:18,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:11:18,715 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:11:19,152 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:21,153 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:23,154 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:25,155 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:27,156 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:29,156 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:31,157 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:33,158 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:34,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:11:34,053 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:11:35,159 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:37,160 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:39,161 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:41,162 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:43,163 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:43,836 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:11:45,164 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:47,164 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:49,165 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:49,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:11:49,292 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:11:51,166 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:53,167 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:55,168 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:57,169 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:11:59,170 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:01,171 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:03,171 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:04,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:12:04,517 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:12:05,172 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:07,173 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:09,174 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:11,175 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:13,176 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:13,916 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:12:15,177 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:17,178 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:19,179 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:19,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:12:19,814 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:12:21,180 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:23,181 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:25,181 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:27,182 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:29,183 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:31,184 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:33,185 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:35,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:12:35,056 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:12:35,186 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:37,186 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:39,187 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:41,188 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:43,189 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:43,996 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:12:45,190 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:47,191 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:49,192 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:50,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:12:50,323 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:12:51,192 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:53,193 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:55,194 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:57,195 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:12:59,196 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:01,197 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:03,198 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:05,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:13:05,640 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:13:06,199 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:08,200 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:10,201 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:12,201 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:14,074 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:13:14,202 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:16,203 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:18,204 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:20,204 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:20,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:13:20,900 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:13:22,205 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:24,206 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:26,207 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:28,207 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:30,208 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:32,209 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:34,210 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:36,210 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:36,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:13:36,352 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:13:38,211 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:40,212 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:42,213 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:44,152 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:13:44,213 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:46,214 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:48,215 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:50,216 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:51,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:13:51,613 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:13:52,217 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:54,217 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:56,218 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:13:58,219 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:00,220 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:02,220 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:04,221 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:06,222 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:06,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:14:06,838 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:14:08,223 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:10,224 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:12,225 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:14,226 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:14,232 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:14:16,226 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:18,227 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:20,228 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:22,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:14:22,156 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:14:22,228 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:24,229 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:26,230 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:29,231 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:31,232 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:33,232 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:35,233 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:37,234 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:37,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:14:37,387 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:14:39,234 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:41,235 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:43,236 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:44,312 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:14:45,237 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:47,238 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:49,239 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:51,240 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:52,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:14:52,619 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:14:53,240 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:55,241 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:57,242 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:14:59,243 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:01,244 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:03,244 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:05,245 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:07,246 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:07,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:15:07,876 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:15:09,247 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:11,248 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:13,248 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:14,391 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:15:15,249 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:17,250 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:19,251 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:21,252 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:23,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:15:23,185 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:15:23,253 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:25,254 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:27,255 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:29,256 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:31,256 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:33,258 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:35,259 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:37,259 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:38,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:15:38,411 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:15:39,260 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:41,261 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:43,262 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:44,472 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:15:45,263 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:47,264 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:49,265 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:51,265 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:53,266 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:53,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:15:53,713 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:15:55,267 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:57,268 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:15:59,268 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:01,269 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:04,270 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:06,271 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:08,272 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:09,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:16:09,016 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:16:10,273 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:12,274 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:14,275 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:14,552 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:16:16,275 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:18,276 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:20,277 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:22,278 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:24,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:16:24,242 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:16:24,279 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:26,280 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:28,280 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:30,281 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:32,282 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:34,283 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:36,284 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:38,285 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:39,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:16:39,587 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:16:40,286 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:42,286 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:44,287 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:44,629 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:16:46,288 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:48,289 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:50,290 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:52,290 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:54,291 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:54,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:16:54,914 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:16:56,292 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:16:58,293 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:00,294 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:02,295 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:04,295 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:06,296 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:08,297 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:10,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:17:10,136 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:17:10,298 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:12,299 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:14,300 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:14,708 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:17:16,301 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:18,301 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:20,302 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:22,303 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:24,304 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:25,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:17:25,446 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:17:26,305 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:28,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:30,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:32,307 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:34,308 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:36,309 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:38,310 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:40,311 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:40,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:17:40,666 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:17:43,312 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:44,786 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:17:45,313 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:47,314 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:49,315 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:51,316 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:53,317 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:55,317 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:55,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:17:55,990 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:17:57,318 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:17:59,319 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:01,320 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:03,321 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:05,322 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:07,322 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:09,323 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:11,324 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:11,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:18:11,341 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:18:13,325 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:14,865 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:18:15,326 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:17,326 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:19,327 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:21,328 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:23,329 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:25,329 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:26,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:18:26,568 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:18:27,330 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:29,331 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:31,332 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:33,333 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:35,334 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:37,334 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:39,335 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:41,336 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:41,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:18:41,920 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:18:43,337 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:44,945 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:18:45,338 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:47,339 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:49,339 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:51,340 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:53,341 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:55,341 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:57,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:18:57,160 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:18:57,342 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:18:59,343 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:01,344 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:03,344 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:05,345 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:08,347 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:10,347 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:12,348 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:12,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:19:12,389 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:19:14,349 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:15,024 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:19:16,350 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:18,351 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:20,352 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:22,353 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:24,353 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:26,354 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:27,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:19:27,628 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:19:28,355 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:30,356 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:32,357 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:34,357 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:36,358 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:38,359 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:40,360 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:42,361 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:42,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:19:42,883 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:19:44,362 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:45,102 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:19:46,362 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:48,363 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:50,364 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:52,365 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:54,366 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:56,367 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:19:58,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:19:58,114 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:19:58,368 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:00,369 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:02,370 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:04,371 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:06,371 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:08,372 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:10,373 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:12,374 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:13,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:20:13,504 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:20:14,374 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:15,182 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:20:16,375 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:18,376 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:20,377 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:22,377 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:24,378 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:26,379 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:28,380 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:28,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:20:28,726 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:20:30,381 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:32,381 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:34,382 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:36,383 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:39,384 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:41,385 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:43,386 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:44,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:20:44,003 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:20:45,261 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:20:45,386 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:47,388 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:49,389 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:51,389 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:53,390 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:55,391 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:57,392 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:20:59,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:20:59,329 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:20:59,393 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:01,394 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:03,395 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:05,396 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:07,396 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:09,397 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:11,398 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:13,399 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:14,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:21:14,552 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:21:15,340 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:21:15,400 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:17,401 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:19,401 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:21,402 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:23,403 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:25,404 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:27,405 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:29,405 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:29,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:21:29,853 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:21:31,406 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:33,407 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:35,408 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:37,409 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:39,409 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:41,410 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:43,411 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:45,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:21:45,138 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:21:45,412 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:45,418 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:21:47,413 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:49,414 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:51,415 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:53,415 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:55,416 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:57,417 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:21:59,418 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:00,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:22:00,445 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:22:01,419 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:03,420 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:05,420 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:07,421 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:09,422 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:11,423 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:13,424 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:15,425 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:15,497 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:22:15,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:22:15,674 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:22:17,426 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:19,426 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:21,427 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:23,428 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:26,430 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:28,430 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:30,431 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:30,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:22:30,979 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:22:32,432 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:34,433 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:36,434 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:38,435 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:40,435 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:42,436 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:44,437 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:45,577 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:22:46,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:22:46,212 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:22:46,438 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:48,439 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:50,440 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:52,441 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:54,441 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:56,442 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:22:58,443 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:00,444 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:01,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:23:01,493 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:23:02,445 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:04,446 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:06,447 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:08,448 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:10,448 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:12,449 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:14,450 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:15,656 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:23:16,451 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:16,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:23:16,719 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:23:18,452 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:20,453 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:22,454 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:24,454 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:26,455 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:29,456 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:31,457 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:32,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:23:32,186 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:23:33,458 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:35,459 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:37,460 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:39,461 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:41,461 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:43,462 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:45,463 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:45,735 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:23:47,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:23:47,414 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:23:47,464 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:49,465 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:51,465 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:53,466 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:55,467 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:57,468 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:23:59,468 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:01,469 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:02,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:24:02,784 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:24:03,470 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:05,471 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:07,472 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:09,472 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:11,473 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:13,474 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:15,475 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:15,815 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:24:17,475 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:18,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:24:18,042 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:24:19,476 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:21,477 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:23,478 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:25,479 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:27,479 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:29,480 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:31,481 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:33,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:24:33,433 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:24:33,481 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:36,483 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:38,483 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:40,484 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:42,485 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:44,486 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:45,895 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:24:46,486 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:48,487 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:48,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:24:48,657 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:24:50,488 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:52,489 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:54,490 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:56,490 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:24:58,491 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:00,492 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:02,492 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:03,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:25:03,957 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:25:04,493 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:06,494 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:08,495 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:10,495 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:12,496 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:14,497 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:15,974 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:25:16,498 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:18,498 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:19,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:25:19,178 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:25:20,499 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:22,500 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:24,501 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:26,501 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:28,502 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:30,503 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:32,504 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:34,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:25:34,409 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:25:34,505 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:36,505 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:38,506 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:40,507 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:42,508 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:44,509 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:46,053 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:25:46,509 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:49,510 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:49,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:25:49,697 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:25:51,511 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:53,512 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:55,513 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:57,513 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:25:59,514 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:01,515 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:03,516 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:05,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:26:05,007 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:26:05,517 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:07,518 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:09,518 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:11,519 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:13,520 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:15,521 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:16,132 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:26:17,521 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:19,522 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:20,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:26:20,232 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:26:21,523 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:23,524 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:25,524 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:27,525 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:29,526 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:31,527 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:33,527 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:35,528 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:35,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:26:35,529 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:26:37,529 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:39,530 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:41,530 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:43,531 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:45,532 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:46,211 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:26:47,533 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:49,533 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:50,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:26:50,753 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:26:51,534 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:53,535 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:55,536 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:57,537 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:26:59,538 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:01,539 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:03,540 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:05,541 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:06,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:27:06,025 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:27:07,542 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:09,543 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:11,544 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:14,545 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:16,290 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:27:16,546 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:18,547 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:20,548 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:21,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:27:21,326 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:27:22,551 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:24,552 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:26,553 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:28,554 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:30,555 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:32,556 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:34,557 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:36,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:27:36,556 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:27:36,561 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:38,562 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:40,563 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:42,564 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:44,564 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:46,369 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:27:46,565 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:48,566 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:50,567 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:51,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:27:51,812 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:27:52,568 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:54,569 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:56,569 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:27:58,570 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:00,571 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:02,571 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:04,572 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:06,573 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:07,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:28:07,071 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:28:08,574 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:10,575 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:12,576 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:14,577 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:16,448 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:28:16,578 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:18,579 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:20,580 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:22,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:28:22,349 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:28:22,581 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:24,582 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:26,583 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:28,583 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:30,584 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:32,585 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:34,586 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:37,588 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:37,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:28:37,596 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:28:39,589 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:41,590 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:43,591 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:45,592 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:46,528 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:28:47,593 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:49,594 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:51,595 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:52,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:28:52,819 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:28:53,596 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:55,597 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:57,598 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:28:59,599 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:01,600 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:03,601 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:05,602 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:07,603 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:08,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:29:08,081 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:29:09,604 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:11,605 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:13,606 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:15,607 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:16,613 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:29:17,608 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:19,609 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:21,610 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:23,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:29:23,441 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:29:23,611 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:25,612 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:27,613 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:29,614 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:31,615 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:33,616 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:35,617 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:37,617 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:38,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:29:38,753 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:29:39,618 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:41,619 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:43,621 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:45,621 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:46,693 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:29:47,622 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:49,623 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:51,624 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:53,625 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:54,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:29:54,056 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:29:55,626 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:29:57,627 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:00,628 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:02,628 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:04,629 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:06,630 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:08,631 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:09,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:30:09,330 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:30:10,632 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:12,632 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:14,633 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:16,634 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:16,772 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:30:18,635 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:20,635 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:22,636 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:24,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:30:24,620 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:30:24,637 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:26,638 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:28,638 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:30,639 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:32,640 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:34,640 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:36,641 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:38,642 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:39,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:30:39,846 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:30:40,643 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:42,643 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:44,644 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:46,645 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:46,851 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:30:48,646 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:50,647 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:52,647 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:54,648 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:55,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:30:55,067 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:30:56,649 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:30:58,650 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:00,651 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:02,652 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:04,653 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:06,654 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:08,655 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:10,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:31:10,387 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:31:10,656 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:12,657 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:14,658 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:16,930 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:31:17,659 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:19,660 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:21,661 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:23,662 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:25,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:31:25,614 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:31:25,663 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:27,664 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:29,665 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:31,666 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:33,667 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:35,668 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:37,669 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:39,670 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:40,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:31:40,920 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:31:41,671 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:43,672 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:45,673 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:47,009 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:31:47,674 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:49,675 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:51,676 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:53,677 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:55,677 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:56,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:31:56,220 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:31:57,678 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:31:59,679 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:01,680 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:03,681 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:05,682 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:07,683 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:09,684 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:11,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:32:11,521 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:32:11,684 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:13,685 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:15,686 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:17,088 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:32:17,687 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:19,687 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:21,688 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:23,689 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:25,690 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:26,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:32:26,748 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:32:27,690 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:29,691 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:31,692 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:33,692 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:35,693 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:37,694 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:39,695 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:41,695 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:41,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:32:41,971 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:32:43,696 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:45,697 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:47,167 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:32:47,698 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:49,698 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:51,699 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:54,700 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:56,701 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:32:57,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:32:57,242 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:32:58,702 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:00,703 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:02,703 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:04,704 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:06,705 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:08,706 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:10,707 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:12,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:33:12,466 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:33:12,708 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:14,709 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:16,710 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:17,247 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:33:18,711 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:20,712 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:22,713 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:24,714 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:26,714 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:27,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:33:27,767 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:33:28,715 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:30,716 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:32,717 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:34,717 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:36,718 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:38,719 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:40,720 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:42,721 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:43,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:33:43,052 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:33:44,721 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:46,722 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:47,326 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:33:48,723 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:50,724 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:52,725 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:54,726 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:56,726 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:33:58,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:33:58,279 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:33:58,727 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:00,728 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:02,729 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:04,729 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:06,730 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:08,731 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:10,732 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:12,733 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:13,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:34:13,574 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:34:14,734 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:16,734 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:17,406 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:34:18,735 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:21,737 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:23,738 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:25,738 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:27,739 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:28,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:34:28,868 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:34:29,740 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:31,741 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:33,741 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:35,742 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:37,743 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:39,744 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:41,745 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:43,745 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:44,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:34:44,132 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:34:45,746 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:47,485 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:34:47,747 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:49,748 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:51,749 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:53,749 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:55,750 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:57,751 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:34:59,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:34:59,379 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:34:59,752 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:01,752 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:03,753 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:05,754 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:07,755 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:09,756 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:11,756 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:13,757 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:14,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:35:14,696 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:35:15,758 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:17,565 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:35:17,758 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:19,759 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:21,760 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:23,761 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:25,761 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:27,762 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:29,763 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:29,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:35:29,919 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:35:31,764 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:33,764 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:35,765 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:38,766 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:40,767 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:42,768 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:44,769 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:45,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:35:45,139 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:35:46,770 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:47,644 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:35:48,770 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:50,771 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:52,772 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:54,773 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:56,774 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:35:58,775 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:00,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:36:00,368 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:36:00,776 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:02,776 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:04,777 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:06,778 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:08,778 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:10,779 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:12,780 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:14,781 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:15,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:36:15,614 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:36:16,782 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:17,723 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:36:18,782 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:20,783 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:22,784 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:24,784 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:26,785 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:28,786 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:30,787 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:30,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:36:30,834 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:36:32,787 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:34,788 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:36,789 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:38,790 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:40,790 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:42,791 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:44,792 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:46,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:36:46,127 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:36:46,793 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:47,804 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:36:48,793 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:50,794 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:52,795 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:55,796 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:57,797 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:36:59,798 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:01,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:37:01,453 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:37:01,799 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:03,800 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:05,800 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:07,801 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:09,802 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:11,803 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:13,804 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:15,805 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:16,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:37:16,682 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:37:17,805 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:17,883 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:37:19,806 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:21,807 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:23,808 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:25,809 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:27,809 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:29,810 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:31,811 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:32,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:37:32,053 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:37:33,812 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:35,812 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:37,813 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:39,814 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:41,815 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:43,815 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:45,816 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:47,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:37:47,297 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:37:47,817 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:47,961 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:37:49,818 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:51,818 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:53,819 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:55,820 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:57,821 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:37:59,822 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:01,823 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:02,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:38:02,668 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:38:03,823 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:05,824 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:07,825 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:09,826 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:11,826 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:13,827 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:15,828 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:17,829 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:17,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:38:17,892 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:38:18,111 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:38:19,829 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:21,830 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:23,831 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:25,832 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:27,832 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:29,833 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:32,834 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:33,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:38:33,188 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:38:34,836 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:36,836 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:38,837 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:40,838 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:42,838 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:44,839 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:46,840 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:48,122 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:38:48,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:38:48,480 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:38:48,841 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:50,842 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:52,842 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:54,843 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:56,844 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:38:58,845 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:00,846 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:02,846 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:03,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:39:03,706 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:39:04,847 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:06,848 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:08,849 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:10,850 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:12,850 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:14,851 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:16,852 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:18,201 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:39:18,852 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:18,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:39:18,967 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:39:20,853 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:22,854 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:24,855 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:26,856 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:28,857 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:30,857 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:32,858 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:34,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:39:34,203 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:39:34,859 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:36,860 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:38,860 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:40,861 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:42,862 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:44,863 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:46,864 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:48,282 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:39:48,864 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:49,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:39:49,443 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:39:50,865 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:52,866 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:54,867 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:57,868 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:39:59,868 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:01,869 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:03,870 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:04,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:40:04,670 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:40:05,871 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:07,872 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:09,872 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:11,873 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:13,874 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:15,875 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:17,875 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:18,361 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:40:19,876 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:19,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:40:19,982 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:40:21,877 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:23,878 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:25,878 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:27,879 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:29,880 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:31,880 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:33,881 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:35,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:40:35,380 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:40:35,882 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:37,883 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:39,884 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:41,884 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:43,885 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:45,886 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:47,887 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:48,441 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:40:49,887 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:50,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:40:50,669 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:40:51,888 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:53,889 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:55,890 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:57,891 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:40:59,892 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:01,892 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:03,893 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:05,894 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:05,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:41:05,898 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:41:07,895 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:09,896 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:11,896 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:13,897 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:15,898 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:18,519 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:41:18,899 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:20,900 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:21,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:41:21,124 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:41:22,901 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:24,901 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:26,902 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:28,903 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:30,904 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:32,905 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:34,905 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:36,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:41:36,343 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:41:36,906 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:38,907 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:40,907 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:42,908 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:44,909 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:46,910 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:48,601 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:41:48,911 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:50,912 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:51,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:41:51,561 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:41:52,912 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:54,913 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:56,914 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:41:58,915 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:00,916 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:02,917 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:04,918 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:06,919 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:06,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:42:06,968 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:42:08,920 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:10,921 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:12,921 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:14,922 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:16,923 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:18,681 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:42:18,923 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:20,924 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:22,925 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:23,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:42:23,829 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:42:24,926 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:26,927 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:28,928 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:30,929 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:32,930 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:34,930 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:36,931 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:38,932 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:39,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:42:39,055 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:42:40,933 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:42,934 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:44,935 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:46,936 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:48,761 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:42:48,937 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:50,937 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:52,938 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:54,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:42:54,277 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:42:54,939 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:56,940 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:42:58,941 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:01,942 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:03,943 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:05,944 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:07,944 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:09,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:43:09,611 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:43:09,945 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:11,946 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:13,947 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:15,948 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:17,949 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:18,841 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:43:19,950 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:21,950 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:23,951 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:24,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:43:24,886 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:43:25,952 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:27,953 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:29,954 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:31,955 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:33,955 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:35,956 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:37,957 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:39,957 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:40,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:43:40,139 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:43:41,958 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:43,959 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:45,959 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:47,960 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:48,921 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:43:49,961 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:51,962 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:53,963 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:55,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:43:55,375 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:43:55,964 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:57,965 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:43:59,966 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:01,967 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:03,968 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:05,969 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:07,970 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:09,971 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:10,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:44:10,679 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:44:11,972 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:13,973 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:15,974 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:17,975 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:19,000 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:44:19,976 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:21,977 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:23,978 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:25,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:44:25,901 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:44:25,979 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:27,980 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:29,981 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:31,981 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:33,982 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:35,983 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:37,984 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:39,985 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:41,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:44:41,127 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:44:42,987 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:44,988 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:46,989 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:48,990 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:49,078 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:44:50,991 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:52,991 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:54,992 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:56,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:44:56,344 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:44:56,993 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:44:58,994 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:00,995 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:02,996 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:04,997 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:06,998 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:08,999 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:11,000 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:11,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:45:11,570 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:45:13,001 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:15,001 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:17,002 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:19,003 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:19,157 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:45:21,004 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:23,005 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:25,006 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:26,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:45:26,914 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:45:27,007 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:29,008 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:31,009 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:33,010 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:35,011 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:37,012 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:39,013 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:41,014 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:42,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:45:42,162 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:45:43,015 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:45,016 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:47,017 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:49,018 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:49,237 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:45:51,019 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:53,020 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:55,021 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:57,022 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:45:57,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:45:57,422 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:45:59,023 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:46:01,024 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:46:03,025 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:46:05,026 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:46:07,027 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:46:09,028 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:46:11,029 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:46:12,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:46:12,708 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:46:13,030 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:46:15,031 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:46:17,031 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:46:19,032 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:46:19,315 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:46:21,033 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:46:23,034 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:46:26,036 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:46:28,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:46:28,009 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:46:28,037 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:46:30,038 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:46:32,039 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:46:34,040 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:46:36,041 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:46:38,042 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:46:43,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:46:43,252 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:46:49,396 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:46:58,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:46:58,476 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:47:13,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:47:13,718 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:47:19,474 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:47:28,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:47:28,986 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:47:44,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:47:44,251 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:47:49,552 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:47:51,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:47:51,792 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 11:47:51,802 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 11:47:51,808 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 11:47:51,813 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 11:47:51,819 DEBUG SenderThread:2194005 [sender.py:send():232] send: metric -2022-10-25 11:47:51,824 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 11:47:51,845 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 11:47:51,927 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 11:47:52,071 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 11:47:54,072 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:47:59,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:47:59,502 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:48:00,075 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:48:11,080 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/config.yaml -2022-10-25 11:48:14,082 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:48:14,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:48:14,730 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:48:19,661 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:48:29,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:48:29,962 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:48:45,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:48:45,191 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:48:49,745 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:49:00,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:49:00,415 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:49:15,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:49:15,645 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:49:19,830 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:49:30,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:49:30,869 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:49:46,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:49:46,098 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:49:49,916 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:50:01,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:50:01,331 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:50:16,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:50:16,565 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:50:20,000 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:50:31,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:50:31,812 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:50:47,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:50:47,041 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:50:50,085 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:51:02,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:51:02,268 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:51:17,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:51:17,496 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:51:20,169 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:51:32,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:51:32,723 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:51:47,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:51:47,949 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:51:50,254 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:52:03,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:52:03,179 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:52:18,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:52:18,408 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:52:20,339 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:52:33,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:52:33,633 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:52:48,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:52:48,860 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:52:50,424 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:53:04,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:53:04,086 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:53:19,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:53:19,315 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:53:20,509 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:53:28,245 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:53:30,245 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:53:32,246 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:53:34,247 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:53:34,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:53:34,560 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:53:36,247 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:53:38,248 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:53:40,249 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:53:42,250 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:53:44,251 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:53:46,252 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:53:48,253 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:53:49,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:53:49,780 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:53:50,254 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:53:50,589 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:53:52,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:52,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:52,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:52,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:52,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:52,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:52,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:52,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:52,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:52,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:52,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:52,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:52,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:52,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:52,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:52,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:52,254 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:53:53,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:53,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:53,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:53,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:53,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:53,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:53,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:53,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:53,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:53,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:53,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:53,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:53,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:53,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:53,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:53,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:54,255 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:53:56,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,256 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:53:56,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:56,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:57,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:57,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:57,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:57,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:57,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:57,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:57,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:57,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:57,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:57,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:57,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:57,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:57,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:57,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:57,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:57,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:58,257 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:53:58,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:58,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:58,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:58,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:58,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:58,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:58,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:58,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:58,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:58,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:58,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:58,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:58,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:58,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:58,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:58,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:53:59,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:00,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,260 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:54:01,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:01,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:02,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,261 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:54:03,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:03,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:04,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:54:05,005 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:54:05,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:05,262 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:54:06,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:06,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,263 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:54:07,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:07,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:08,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:08,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:08,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:08,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:08,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:08,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:08,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:08,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:08,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:08,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:08,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:08,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:08,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:08,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:08,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:08,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,265 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:54:09,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:09,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:10,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:10,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:10,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:10,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:10,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:10,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:10,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:10,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:10,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:10,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:10,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:10,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:10,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:10,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:10,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:10,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:11,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:11,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:11,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:11,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:11,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:11,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:11,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:11,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:11,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:11,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:11,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:11,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:11,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:11,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:11,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:11,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:11,265 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:54:12,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:12,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:12,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:12,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:12,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:12,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:12,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:12,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:12,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:12,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:12,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:12,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:12,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:12,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:12,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:12,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:54:13,266 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:54:15,267 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:54:19,269 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:54:20,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:54:20,349 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:54:20,670 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:54:25,272 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:54:27,273 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:54:31,275 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:54:35,276 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:54:35,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:54:35,573 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:54:39,278 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:54:41,279 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:54:45,281 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:54:47,282 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:54:50,750 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:54:50,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:54:50,798 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:54:51,284 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:54:53,284 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:54:57,286 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:54:59,287 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:01,288 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:05,289 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:06,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:55:06,013 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:55:07,290 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:09,291 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:11,292 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:13,293 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:14,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:14,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:14,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:14,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:14,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:14,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:14,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:14,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:14,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:14,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:14,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:14,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:17,295 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:19,296 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:20,829 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:55:21,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:55:21,236 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:55:21,296 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:23,297 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:25,298 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:27,299 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:29,300 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:31,301 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:33,301 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:35,302 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:36,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:55:36,543 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:55:37,303 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:39,304 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:41,305 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:43,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:45,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:45,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:45,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:46,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:47,309 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:47,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:47,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:47,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:47,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:47,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:47,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:47,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:47,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:47,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:47,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:47,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:47,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:47,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:47,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:47,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:47,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:48,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,309 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:49,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:49,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:50,904 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:55:51,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,310 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:51,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:55:51,796 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:55:51,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:51,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:52,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,311 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:53,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:53,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:54,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,311 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:55,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:55,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:56,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,312 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:57,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:57,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:58,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:59,313 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:55:59,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:59,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:59,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:59,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:59,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:59,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:59,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:59,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:59,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:59,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:59,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:59,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:59,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:59,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:59,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:55:59,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:00,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:00,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:00,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:00,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:00,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:00,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:00,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:00,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:00,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:00,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:00,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:00,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:00,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:00,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:00,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:00,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:01,314 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:56:03,315 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:56:03,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:03,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:03,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:03,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:03,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:03,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:03,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:03,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:03,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:03,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:03,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:03,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:03,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:03,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:03,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:03,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:05,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:05,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:05,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:05,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:05,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:05,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:05,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:05,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:05,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:05,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:05,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:05,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:05,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:05,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:05,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:05,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:07,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:56:07,021 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:56:07,316 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:56:12,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:12,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:12,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:12,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:12,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:12,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:12,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:12,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:12,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:12,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:12,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:12,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:12,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:12,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:12,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:12,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:56:14,319 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:56:18,321 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:56:20,322 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:56:20,982 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:56:22,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:56:22,246 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:56:24,324 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:56:28,326 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:56:32,328 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:56:34,329 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:56:37,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:56:37,618 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:56:38,331 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:56:40,332 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:56:44,334 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:56:46,335 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:56:48,336 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:56:51,061 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:56:52,338 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:56:52,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:56:52,842 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:56:54,339 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:56:56,340 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:56:58,341 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:00,342 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:03,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:03,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:03,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:03,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:03,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:03,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:03,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:03,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:03,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:03,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:03,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:03,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:04,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:04,224 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 11:57:04,449 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 11:57:04,470 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:04,522 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 11:57:05,465 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 11:57:06,466 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:07,466 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:08,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:57:08,066 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:57:08,467 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:10,468 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:12,469 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:14,470 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:16,471 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:18,472 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:20,472 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:21,165 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:57:22,473 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:23,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:57:23,348 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:57:24,474 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:26,475 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:28,476 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:30,477 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:32,478 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:34,479 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:36,480 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:37,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:37,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:38,481 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:38,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:57:38,623 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:57:39,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:39,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:39,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:39,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:39,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:39,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:39,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:39,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:39,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:39,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:39,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:39,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:39,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:39,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:39,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:39,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,482 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:40,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:40,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:41,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,483 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:42,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:42,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:43,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,484 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:44,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:44,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:45,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,485 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:46,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:46,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:47,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,486 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:48,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:48,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:49,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:50,487 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:51,247 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:57:52,488 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:53,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:57:53,873 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:57:56,489 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:57:57,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:57,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:57,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:57,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:57,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:57,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:57,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:57,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:57,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:57,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:57,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:57,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:57,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:57,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:57,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:57:57,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:00,491 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:58:01,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:01,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:01,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:01,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:01,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:01,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:01,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:01,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:01,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:01,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:01,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:01,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:01,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:01,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:01,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:01,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:03,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:03,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:03,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:03,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:03,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:03,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:03,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:03,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:03,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:03,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:03,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:03,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:03,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:03,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:03,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:03,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:04,493 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:58:08,495 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:58:08,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:08,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:08,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:08,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:08,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:08,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:08,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:08,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:08,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:08,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:08,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:08,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:08,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:08,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:08,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:08,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:09,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:58:09,093 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:58:12,497 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:58:16,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:16,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:16,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:16,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:16,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:16,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:16,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:16,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:16,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:16,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:16,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:16,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:16,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:16,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:16,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:16,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:16,499 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:58:18,500 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:58:21,327 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:58:23,502 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:58:24,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:58:24,417 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:58:27,504 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:58:29,505 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:58:33,506 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:58:35,507 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:58:37,508 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:58:39,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:58:39,709 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:58:41,510 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:58:43,511 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:58:45,512 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:58:47,513 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:58:51,407 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:58:51,515 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:58:52,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:52,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:52,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:52,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:52,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:52,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:52,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:52,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:52,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:52,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:52,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:52,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:58:53,516 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:58:54,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:58:54,934 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:58:55,517 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:58:57,518 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:58:59,519 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:59:01,520 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:59:03,521 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:59:05,522 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:59:07,523 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:59:09,524 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:59:10,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:59:10,159 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:59:11,524 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:59:13,525 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:59:15,526 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:59:17,527 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:59:19,528 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:59:21,486 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:59:21,529 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:59:23,530 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:59:25,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:59:25,497 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:59:25,531 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:59:27,532 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:59:28,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:28,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:28,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:28,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:28,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:28,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:28,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:28,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:28,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:28,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:28,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:28,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:28,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:28,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:28,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:28,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:29,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:29,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:29,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:29,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:29,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:29,534 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:59:29,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:29,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:29,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:29,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:29,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:29,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:29,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:29,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:29,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:29,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:29,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:30,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,535 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:59:31,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:31,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:32,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,536 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:59:33,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:33,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:34,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,536 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:59:35,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:35,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:36,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,537 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:59:37,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:37,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:39,538 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:59:40,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:40,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:59:40,721 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:59:42,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:42,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:42,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:42,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:42,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:42,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:42,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:42,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:42,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:42,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:42,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:42,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:42,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:42,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:42,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:42,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,541 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:59:45,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:45,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:47,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:48,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:48,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:48,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:48,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:48,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:48,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:48,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:48,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:48,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:48,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:48,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:48,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:48,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:48,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:48,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:48,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:49,543 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:59:51,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 11:59:51,571 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 11:59:53,544 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:59:55,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 11:59:55,950 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 11:59:57,546 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 11:59:59,547 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:00:01,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:01,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:01,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:01,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:01,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:01,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:01,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:01,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:01,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:01,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:01,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:01,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:01,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:01,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:01,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:01,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:04,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:04,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:04,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:04,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:04,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:04,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:04,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:04,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:04,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:04,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:04,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:04,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:04,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:04,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:04,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:04,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:05,550 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:00:07,551 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:00:11,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:00:11,167 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:00:11,552 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:00:13,553 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:00:15,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:15,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:15,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:15,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:15,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:15,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:15,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:15,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:15,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:15,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:15,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:15,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:15,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:15,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:15,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:15,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:17,555 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:00:19,556 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:00:21,650 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:00:24,558 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:00:26,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:00:26,439 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:00:26,559 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:00:28,560 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:00:32,562 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:00:34,563 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:00:36,564 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:00:38,565 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:00:41,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:41,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:41,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:41,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:41,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:41,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:41,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:41,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:41,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:41,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:41,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:41,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:41,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:00:41,664 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:00:41,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:00:41,948 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 12:00:42,140 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 12:00:42,218 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 12:00:42,566 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:00:42,567 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 12:00:43,567 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:00:44,567 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:00:46,568 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:00:48,569 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:00:50,570 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:00:51,746 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:00:52,571 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:00:54,572 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:00:56,573 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:00:57,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:00:57,033 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:00:58,574 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:01:00,575 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:01:02,576 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:01:04,577 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:01:06,578 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:01:08,579 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:01:10,580 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:01:12,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:01:12,288 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:01:12,581 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:01:14,581 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:01:16,582 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:01:18,583 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:01:20,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:20,584 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:01:21,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:21,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:21,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:21,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:21,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:21,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:21,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:21,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:21,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:21,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:21,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:21,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:21,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:21,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:21,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:21,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:21,824 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:01:22,585 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:01:23,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:23,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:24,620 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:01:25,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:25,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,621 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:01:26,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:26,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:01:27,513 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:01:27,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:27,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:28,622 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:01:30,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,623 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:01:30,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:30,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:32,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:34,625 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:01:35,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:35,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:35,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:35,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:35,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:35,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:35,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:35,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:35,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:35,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:35,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:35,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:35,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:35,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:35,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:35,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:36,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:36,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:36,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:36,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:36,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:36,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:36,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:36,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:36,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:36,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:36,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:36,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:36,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:36,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:36,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:36,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:38,626 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:01:39,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:39,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:42,628 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:01:42,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:01:42,741 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:01:43,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:43,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:44,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:44,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:44,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:44,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:44,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:44,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:44,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:44,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:44,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:44,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:44,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:44,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:44,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:44,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:44,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:44,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:44,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:44,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:44,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:44,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:44,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:44,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:44,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:44,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:45,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:46,630 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:01:47,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:47,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:48,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:48,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:48,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:48,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:48,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:48,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:48,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:48,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:48,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:48,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:48,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:48,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:48,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:48,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:48,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:48,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:50,631 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:01:51,908 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:01:54,633 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:01:56,634 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:01:57,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:57,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:01:58,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:01:58,100 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:02:00,636 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:02:04,638 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:02:06,639 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:02:11,640 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:02:12,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:12,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:12,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:12,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:12,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:12,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:12,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:12,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:12,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:12,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:12,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:12,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:12,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:12,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:12,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:12,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:13,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:02:13,330 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:02:13,641 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:02:15,642 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:02:19,644 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:02:21,645 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:02:21,988 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:02:23,646 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:02:25,647 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:02:28,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:02:28,560 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:02:29,649 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:02:30,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:30,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:30,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:30,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:31,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:31,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:31,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:31,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:31,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:31,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:31,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:31,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:02:31,650 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:02:33,651 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:02:35,651 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:02:37,652 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:02:39,653 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:02:41,654 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:02:43,655 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:02:43,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:02:43,791 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:02:45,656 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:02:47,657 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:02:49,658 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:02:51,659 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:02:52,069 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:02:53,660 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:02:55,661 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:02:57,662 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:02:59,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:02:59,009 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:02:59,662 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:03:01,663 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:03:03,664 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:03:05,665 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:03:07,666 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:03:09,667 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:03:11,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:11,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:11,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:11,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:11,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:11,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:11,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:11,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:11,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:11,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:11,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:11,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:11,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:11,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:11,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:11,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:11,668 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:03:13,669 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:03:13,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:13,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:13,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:13,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:13,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:13,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:13,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:13,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:13,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:13,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:13,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:13,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:13,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:13,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:13,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:13,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:03:14,231 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:03:14,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:14,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,670 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:03:15,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:15,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:16,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:17,670 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:03:19,671 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:03:20,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:20,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:20,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:20,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:20,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:20,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:20,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:20,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:20,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:20,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:20,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:20,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:20,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:20,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:20,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:20,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:21,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:22,149 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:03:23,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:23,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:23,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:23,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:23,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:23,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:23,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:23,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:23,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:23,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:23,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:23,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:23,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:23,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:23,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:23,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:24,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:24,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:24,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:24,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:24,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:24,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:24,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:24,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:24,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:24,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:24,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:24,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:24,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:24,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:24,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:24,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:25,673 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:03:25,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:25,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:25,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:25,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:25,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:25,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:25,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:25,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:25,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:25,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:25,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:25,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:25,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:25,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:25,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:25,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:26,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:27,674 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:03:27,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:27,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:27,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:27,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:27,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:27,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:27,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:27,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:27,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:27,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:27,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:27,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:27,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:27,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:27,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:27,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:28,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:29,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:03:29,457 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:03:29,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:29,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:29,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:29,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:29,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:29,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:29,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:29,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:29,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:29,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:29,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:29,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:29,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:29,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:29,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:29,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:30,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:30,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:30,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:30,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:30,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:30,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:30,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:30,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:30,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:30,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:30,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:30,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:30,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:30,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:30,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:30,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:31,676 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:03:32,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:32,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:32,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:32,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:32,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:32,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:32,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:32,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:32,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:32,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:32,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:32,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:32,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:32,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:32,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:32,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,678 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:03:35,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:35,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:36,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:36,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:36,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:36,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:36,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:36,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:36,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:36,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:36,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:36,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:36,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:36,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:36,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:36,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:36,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:36,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:37,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,679 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:03:39,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:39,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:41,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:43,681 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:03:44,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:03:44,686 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:03:45,682 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:03:46,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:46,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:46,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:46,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:46,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:46,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:46,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:46,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:46,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:46,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:46,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:46,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:46,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:46,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:46,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:46,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:49,683 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:03:50,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:50,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:03:51,684 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:03:52,227 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:03:55,686 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:03:57,687 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:00,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:04:00,579 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:04:01,689 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:03,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:03,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:03,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:03,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:03,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:03,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:03,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:03,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:03,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:03,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:03,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:03,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:03,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:03,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:03,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:03,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:04,690 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:06,691 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:08,692 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:12,694 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:14,695 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:15,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:04:15,802 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:04:16,696 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:18,697 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:18,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:18,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:18,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:18,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:19,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:19,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:19,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:19,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:19,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:19,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:19,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:19,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:19,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:04:19,734 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 12:04:19,977 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 12:04:20,051 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 12:04:20,802 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:20,802 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 12:04:22,357 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:04:22,803 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:24,804 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:26,805 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:28,806 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:30,807 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:31,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:04:31,161 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:04:32,807 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:34,808 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:36,809 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:38,810 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:40,811 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:42,812 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:44,813 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:46,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:04:46,386 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:04:46,814 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:48,815 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:50,816 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:52,436 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:04:52,817 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:54,818 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:56,818 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:04:58,820 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:05:00,820 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:05:01,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:01,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:01,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:01,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:01,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:01,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:01,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:01,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:01,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:01,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:01,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:01,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:01,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:01,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:01,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:01,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:01,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:05:01,612 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:05:02,821 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:05:04,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:04,822 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:05:05,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:05,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:06,823 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:05:07,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:07,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:08,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:08,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:08,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:08,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:08,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:08,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:08,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:08,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:08,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:08,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:08,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:08,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:08,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:08,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:08,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:08,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:10,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:12,826 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:05:14,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:14,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:16,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:16,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:16,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:16,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:16,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:16,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:16,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:16,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:16,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:16,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:16,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:16,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:16,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:16,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:16,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:16,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:16,828 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:05:16,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:05:16,882 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:05:18,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:18,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:19,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:19,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:19,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:19,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:19,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:19,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:19,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:19,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:19,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:19,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:19,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:19,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:19,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:19,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:19,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:19,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:20,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:20,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:20,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:20,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:20,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:20,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:20,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:20,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:20,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:20,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:20,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:20,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:20,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:20,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:20,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:20,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:20,830 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:05:21,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:21,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:21,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:21,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:21,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:21,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:21,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:21,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:21,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:21,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:21,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:21,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:21,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:21,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:21,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:21,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:22,512 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:05:23,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:23,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:23,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:23,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:23,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:23,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:23,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:23,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:23,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:23,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:23,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:23,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:23,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:23,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:23,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:23,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:24,832 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:05:25,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,448 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:25,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:27,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:27,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:27,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:27,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:27,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:27,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:27,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:27,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:27,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:27,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:27,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:27,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:27,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:27,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:27,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:27,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:28,834 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:05:28,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:28,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:28,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:28,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:28,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:28,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:28,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:28,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:28,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:28,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:28,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:28,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:28,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:28,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:28,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:28,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,834 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:05:30,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:30,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:05:32,253 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:05:32,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:32,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:33,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:33,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:33,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:33,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:33,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:33,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:33,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:33,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:33,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:33,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:33,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:33,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:33,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:33,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:33,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:33,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:34,836 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:05:35,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:35,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:37,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:37,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:37,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:37,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:37,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:37,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:37,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:37,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:37,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:37,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:37,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:37,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:37,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:37,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:37,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:37,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:38,838 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:05:40,839 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:05:42,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:42,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:42,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:42,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:42,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:42,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:42,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:42,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:42,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:42,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:42,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:42,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:42,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:42,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:42,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:42,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:44,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:44,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:44,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:44,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:44,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:44,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:44,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:44,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:44,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:44,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:44,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:44,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:44,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:44,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:44,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:44,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:44,841 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:05:46,842 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:05:47,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:05:47,479 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:05:48,843 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:05:52,571 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:05:52,844 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:05:54,845 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:05:56,846 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:05:57,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:57,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:57,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:57,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:57,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:57,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:57,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:57,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:57,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:57,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:57,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:57,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:57,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:57,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:57,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:57,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:05:58,847 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:02,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:06:02,704 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:06:02,849 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:04,850 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:06,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:06,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:06,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:06,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:06,851 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:07,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:07,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:07,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:07,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:07,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:07,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:07,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:07,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:08,852 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:10,852 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:12,853 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:14,854 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:16,855 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:17,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:06:17,993 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:06:18,856 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:20,857 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:22,626 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:06:22,858 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:24,859 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:26,860 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:28,861 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:30,862 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:32,863 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:33,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:06:33,223 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:06:34,864 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:36,865 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:38,866 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:40,866 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:42,867 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:44,868 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:46,869 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:48,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:06:48,621 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:06:48,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:48,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:48,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:48,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:48,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:48,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:48,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:48,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:48,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:48,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:48,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:48,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:48,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:48,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:48,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:48,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:48,870 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:50,871 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:52,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:52,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:52,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:52,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:52,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:52,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:52,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:52,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:52,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:52,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:52,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:52,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:52,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:52,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:52,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:52,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:52,705 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:06:52,872 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:54,873 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:06:55,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:55,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:55,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:55,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:55,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:55,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:55,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:55,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:55,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:55,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:55,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:55,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:55,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:55,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:55,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:55,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:57,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:58,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:58,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:58,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:58,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:58,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:58,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:58,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:58,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:58,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:58,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:58,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:58,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:58,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:58,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:58,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:58,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:06:58,875 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:07:02,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:02,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:02,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:02,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:02,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:02,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:02,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:02,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:02,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:02,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:02,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:02,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:02,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:02,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:02,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:02,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:03,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:07:03,854 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:07:03,877 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:07:03,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:03,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:04,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:04,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:04,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:04,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:04,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:04,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:04,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:04,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:04,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:04,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:04,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:04,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:04,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:04,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:05,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:05,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:05,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:05,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:05,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:05,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:05,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:05,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:05,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:05,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:05,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:05,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:05,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:05,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:05,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:05,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:06,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:07,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:07,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:07,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:07,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:07,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:07,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:07,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:07,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:07,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:07,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:07,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:07,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:07,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:07,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:07,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:07,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:07,879 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:07:08,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:08,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:11,880 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:07:13,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:13,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:14,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:15,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:15,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:15,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:15,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:15,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:15,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:15,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:15,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:15,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:15,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:15,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:15,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:15,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:15,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:15,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:15,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:15,882 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:07:16,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:16,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,448 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:18,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:07:19,268 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:07:19,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,884 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:07:19,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:19,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:20,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:20,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:20,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:20,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:20,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:20,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:20,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:20,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:20,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:20,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:20,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:20,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:20,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:20,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:20,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:20,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:21,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:21,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:21,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:21,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:21,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:21,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:21,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:21,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:21,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:21,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:21,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:21,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:21,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:21,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:21,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:21,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:21,885 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:07:22,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:22,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:22,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:22,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:22,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:22,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:22,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:22,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:22,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:22,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:22,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:22,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:22,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:22,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:22,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:22,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:22,786 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:07:23,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:23,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:23,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:23,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:23,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:23,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:23,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:23,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:23,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:23,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:23,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:23,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:23,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:23,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:23,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:23,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:24,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:25,886 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:07:26,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:26,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,887 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:07:27,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:27,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:28,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:28,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:28,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:28,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:28,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:28,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:28,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:28,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:30,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:30,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:30,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:30,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:30,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:30,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:30,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:30,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:30,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:30,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:30,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:30,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:30,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:30,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:30,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:30,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:31,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:31,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:31,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:31,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:31,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:31,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:31,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:31,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:31,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:31,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:31,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:31,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:31,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:31,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:31,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:31,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:31,888 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:07:32,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:32,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:32,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:32,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:32,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:32,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:32,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:32,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:32,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:32,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:32,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:32,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:32,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:32,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:32,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:32,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:33,889 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:07:34,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:07:34,506 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:07:34,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:34,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:34,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:34,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:34,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:34,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:34,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:34,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:34,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:34,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:34,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:34,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:34,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:34,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:34,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:34,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:36,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:36,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:36,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:36,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:36,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:36,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:36,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:36,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:36,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:36,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:36,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:36,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:36,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:36,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:36,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:36,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:37,890 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:07:39,891 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:07:41,892 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:07:45,893 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:07:47,894 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:07:49,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:07:49,775 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:07:49,895 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:07:50,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:50,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:50,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:50,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:50,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:50,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:50,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:50,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:50,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:50,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:50,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:50,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:50,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:50,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:50,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:50,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:51,896 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:07:52,865 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:07:53,896 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:07:54,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:54,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:54,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:54,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:55,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:55,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:55,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:55,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:55,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:55,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:55,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:55,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:55,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:07:55,548 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 12:07:55,765 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 12:07:55,842 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 12:07:55,897 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 12:07:57,898 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:07:59,898 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:08:01,899 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:08:03,900 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:08:05,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:08:05,105 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:08:05,902 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:08:07,903 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:08:09,903 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:08:11,904 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:08:13,905 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:08:15,906 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:08:17,907 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:08:19,907 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:08:20,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:08:20,335 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:08:21,908 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:08:23,007 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:08:23,909 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:08:25,910 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:08:27,910 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:08:29,911 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:08:31,912 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:08:33,913 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:08:35,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:08:35,619 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:08:35,913 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:08:37,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:37,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:37,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:37,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:37,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:37,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:37,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:37,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:37,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:37,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:37,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:37,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:37,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:37,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:37,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:37,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:37,914 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:08:39,915 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:08:41,916 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:08:47,918 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:08:48,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:48,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:48,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:48,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:48,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:48,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:48,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:48,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:48,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:48,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:48,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:48,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:48,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:48,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:48,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:48,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:50,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:08:50,909 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:08:51,919 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:08:52,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:52,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:53,089 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:08:53,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:53,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:53,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:53,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:53,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:53,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:53,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:53,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:53,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:53,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:53,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:53,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:53,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:53,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:53,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:53,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:54,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:54,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:54,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:54,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:54,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:54,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:54,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:54,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:54,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:54,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:54,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:54,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:54,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:54,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:54,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:54,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:55,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:55,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:55,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:55,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:55,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:55,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:55,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:55,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:55,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:55,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:55,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:55,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:55,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:55,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:55,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:55,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:55,921 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:08:55,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:55,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:56,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:56,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:56,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:56,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:56,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:56,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:56,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:56,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:56,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:56,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:56,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:56,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:56,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:56,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:57,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:57,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:57,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:57,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:57,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:57,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:57,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:57,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:57,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:57,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:57,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:57,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:57,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:57,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:57,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:57,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:08:59,922 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:09:00,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:00,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:00,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:00,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:00,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:00,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:00,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:00,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:00,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:00,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:00,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:00,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:00,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:00,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:00,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:00,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:01,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:01,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:01,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:01,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:01,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:01,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:01,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:01,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:01,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:01,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:01,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:01,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:01,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:01,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:01,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:01,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:01,923 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:09:02,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:02,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:02,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:02,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:02,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:02,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:02,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:02,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:02,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:02,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:02,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:02,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:02,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:02,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:02,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:02,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:04,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:05,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:05,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:05,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:05,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:05,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:05,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:05,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:05,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:05,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:05,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:05,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:05,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:05,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:05,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:05,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:05,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:05,925 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:09:06,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:09:06,132 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:09:07,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:07,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:09,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:10,928 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:09:11,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:11,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:12,929 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:09:13,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:13,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:14,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,932 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:09:16,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:16,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:17,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:17,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:17,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:17,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:17,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:17,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:17,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:17,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:17,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:17,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:17,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:17,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:17,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:17,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:17,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:17,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:18,933 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:09:19,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:19,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:19,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:19,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:19,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:19,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:19,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:19,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:19,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:19,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:19,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:19,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:19,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:19,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:19,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:19,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:20,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:20,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:20,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:20,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:20,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:20,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:20,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:20,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:20,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:20,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:20,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:20,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:20,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:20,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:20,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:20,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:21,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:09:21,408 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:09:21,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:21,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:21,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:21,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:21,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:21,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:21,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:21,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:21,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:21,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:21,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:21,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:21,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:21,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:21,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:21,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:22,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:22,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:22,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:22,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:22,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:22,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:22,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:22,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:22,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:22,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:22,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:22,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:22,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:22,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:22,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:22,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:22,935 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:09:23,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:23,168 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:09:24,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:24,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:24,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:24,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:24,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:24,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:24,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:24,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:24,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:24,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:24,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:24,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:24,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:24,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:24,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:24,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:24,936 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:09:25,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:25,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:25,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:25,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:25,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:25,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:25,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:25,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:25,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:25,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:25,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:25,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:25,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:25,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:25,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:25,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:27,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:27,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:27,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:27,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:27,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:27,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:27,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:27,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:27,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:27,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:27,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:27,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:27,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:27,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:27,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:27,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:28,938 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:09:30,939 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:09:32,939 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:09:34,940 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:09:36,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:09:36,636 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:09:38,942 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:09:40,943 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:09:42,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:42,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:42,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:42,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:42,944 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:09:43,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:43,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:43,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:43,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:43,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:43,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:43,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:43,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:44,945 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:09:46,946 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:09:48,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:48,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:48,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:48,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:48,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:48,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:48,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:48,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:48,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:48,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:48,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:48,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:48,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:48,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:48,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:48,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:09:48,947 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:09:50,948 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:09:51,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:09:51,859 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:09:53,244 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:09:54,950 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:09:56,951 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:09:58,952 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:10:00,953 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:10:02,954 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:10:06,955 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:10:07,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:10:07,088 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:10:08,956 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:10:10,957 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:10:12,958 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:10:14,959 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:10:16,960 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:10:18,961 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:10:20,962 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:10:22,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:10:22,370 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:10:22,963 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:10:23,318 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:10:24,964 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:10:26,965 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:10:28,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:28,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:28,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:28,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:28,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:28,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:28,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:28,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:28,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:28,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:28,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:28,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:28,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:28,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:28,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:28,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:28,966 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:10:30,967 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:10:32,968 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:10:36,969 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:10:37,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:10:37,745 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:10:40,971 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:10:43,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:43,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:44,972 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:10:47,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:47,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:47,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:47,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:47,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:47,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:47,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:47,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:47,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:47,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:47,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:47,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:47,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:47,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:47,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:47,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:48,974 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:10:51,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:51,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:51,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:51,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:51,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:51,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:51,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:51,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:51,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:51,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:51,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:51,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:51,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:51,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:51,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:51,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:52,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:10:52,966 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:10:52,976 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:10:53,394 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:10:53,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:53,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:53,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:53,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:53,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:53,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:53,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:53,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:53,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:53,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:53,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:53,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:53,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:53,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:53,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:53,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:54,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:54,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:54,448 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:54,448 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:54,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:54,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:54,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:54,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:54,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:54,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:54,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:54,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:54,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:54,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:54,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:54,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:55,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:55,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:55,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:55,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:55,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:55,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:55,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:55,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:55,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:55,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:55,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:55,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:55,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:55,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:55,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:55,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:56,978 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:10:58,979 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:10:59,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:59,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:59,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:59,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:59,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:59,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:59,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:59,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:59,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:59,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:59,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:59,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:59,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:59,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:59,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:10:59,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:00,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:01,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:01,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:01,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:01,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:01,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:01,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:01,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:01,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:01,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:01,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:01,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:01,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:01,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:01,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:01,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:01,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:02,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:02,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:02,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:02,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:02,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:02,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:02,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:02,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:02,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:02,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:02,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:02,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:02,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:02,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:02,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:02,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:02,981 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:11:03,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:03,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:05,982 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:11:06,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:06,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:06,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:06,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:06,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:06,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:06,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:06,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:06,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:06,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:06,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:06,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:06,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:06,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:06,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:06,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:06,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:06,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:06,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:07,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:07,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:07,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:07,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:07,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:07,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:07,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:07,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:07,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:07,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:07,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:07,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:07,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:08,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:11:08,300 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:11:08,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:08,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:08,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:08,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:08,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:08,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:08,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:08,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:08,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:08,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:08,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:08,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:08,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:08,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:08,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:08,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:09,984 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:11:11,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:11,985 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:11:12,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:12,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:13,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:13,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:13,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:13,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:13,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:13,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:13,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:13,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:13,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:13,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:13,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:13,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:13,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:13,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:13,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:13,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:14,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:15,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:15,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:15,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:15,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:15,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:15,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:15,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:15,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:15,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:15,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:15,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:15,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:15,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:15,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:15,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:15,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:15,986 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:11:17,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:17,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:17,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:17,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:17,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:17,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:17,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:17,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:17,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:17,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:17,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:17,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:17,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:17,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:17,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:17,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:17,987 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:11:18,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:18,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:19,988 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:11:22,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:22,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:22,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:22,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:22,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:22,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:22,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:22,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:22,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:22,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:22,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:22,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:22,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:22,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:22,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:22,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:23,471 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:11:23,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:23,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:23,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:23,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:23,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:23,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:23,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:23,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:23,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:23,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:23,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:23,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:23,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:23,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:23,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:23,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:23,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:11:23,525 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:11:23,990 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:11:24,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:24,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:24,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:24,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:24,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:24,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:24,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:24,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:24,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:24,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:24,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:24,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:24,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:24,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:24,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:24,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:25,991 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:11:27,992 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:11:29,993 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:11:31,994 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:11:32,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:32,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:32,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:32,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:33,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:33,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:33,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:33,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:33,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:33,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:33,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:33,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:33,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 12:11:33,371 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 12:11:33,596 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 12:11:33,681 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 12:11:33,995 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 12:11:34,996 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:11:35,996 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:11:37,997 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:11:38,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:11:38,944 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:11:39,998 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:11:41,999 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:11:44,000 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:11:46,001 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:11:48,002 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:11:50,003 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:11:52,004 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:11:53,732 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:11:54,005 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:11:54,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:11:54,172 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:11:56,005 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:11:58,006 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:00,007 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:02,008 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:04,009 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:06,010 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:08,011 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:09,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:12:09,482 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:12:10,012 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:12,013 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:14,014 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:16,015 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:18,016 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:20,016 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:22,017 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:23,810 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:12:24,018 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:24,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:12:24,817 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:12:26,020 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:28,020 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:30,021 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:32,022 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:34,023 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:36,024 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:38,025 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:40,025 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:40,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:12:40,234 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:12:42,026 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:44,027 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:46,028 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:48,029 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:50,030 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:52,031 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:53,888 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:12:54,032 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:55,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:12:55,457 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:12:57,033 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:12:59,034 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:01,035 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:03,035 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:05,036 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:07,037 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:09,038 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:10,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:13:10,679 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:13:11,039 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:13,040 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:15,040 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:17,041 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:19,042 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:21,043 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:23,044 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:23,965 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:13:25,044 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:26,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:13:26,044 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:13:27,049 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:29,050 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:31,051 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:33,052 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:35,052 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:37,053 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:39,054 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:41,055 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:41,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:13:41,268 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:13:43,056 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:45,056 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:47,057 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:49,058 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:51,059 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:53,060 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:54,044 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:13:55,060 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:56,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:13:56,558 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:13:57,061 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:13:59,062 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:01,063 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:03,064 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:05,065 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:07,065 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:09,066 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:11,067 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:11,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:14:11,780 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:14:13,068 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:15,068 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:17,069 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:19,070 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:21,071 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:23,071 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:24,121 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:14:25,072 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:27,073 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:27,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:14:27,142 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:14:29,074 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:31,074 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:33,075 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:35,076 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:37,076 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:39,078 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:42,079 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:42,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:14:42,499 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:14:44,080 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:46,080 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:48,081 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:50,082 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:52,083 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:54,084 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:54,197 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:14:56,084 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:14:57,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:14:57,725 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:14:58,085 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:00,086 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:02,087 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:04,087 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:06,088 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:08,089 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:10,090 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:12,090 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:13,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:15:13,035 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:15:14,091 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:16,092 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:18,093 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:20,094 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:22,095 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:24,096 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:24,274 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:15:26,097 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:28,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:15:28,259 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:15:29,099 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:31,100 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:33,101 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:35,102 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:37,103 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:39,104 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:41,105 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:43,106 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:43,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:15:43,510 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:15:45,107 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:47,108 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:49,109 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:51,110 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:53,111 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:54,355 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:15:55,112 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:57,113 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:15:58,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:15:58,844 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:15:59,114 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:01,115 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:03,116 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:05,117 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:07,118 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:09,119 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:11,120 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:13,121 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:14,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:16:14,073 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:16:15,122 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:17,123 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:19,124 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:21,125 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:23,126 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:24,434 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:16:25,127 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:27,128 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:29,129 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:29,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:16:29,352 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:16:31,130 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:33,131 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:35,132 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:37,133 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:39,134 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:41,135 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:43,136 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:44,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:16:44,655 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:16:45,137 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:47,138 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:49,139 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:51,140 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:53,141 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:54,510 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:16:55,142 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:57,143 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:59,144 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:16:59,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:16:59,889 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:17:01,145 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:03,146 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:06,147 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:08,148 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:10,149 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:12,150 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:14,151 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:15,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:17:15,134 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:17:16,152 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:18,154 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:20,154 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:22,155 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:24,156 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:24,589 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:17:26,157 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:28,158 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:30,159 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:30,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:17:30,403 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:17:32,160 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:34,161 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:36,162 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:38,163 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:40,164 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:42,165 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:44,166 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:45,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:17:45,623 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:17:46,168 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:48,169 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:50,169 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:52,170 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:54,171 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:54,669 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:17:56,172 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:17:58,173 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:00,174 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:00,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:18:00,841 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:18:02,175 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:04,176 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:06,177 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:08,178 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:10,179 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:12,180 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:14,181 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:16,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:18:16,143 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:18:16,182 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:18,183 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:20,184 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:23,185 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:24,748 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:18:25,186 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:27,187 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:29,188 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:31,189 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:31,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:18:31,557 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:18:33,190 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:35,191 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:37,192 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:39,193 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:41,194 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:43,195 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:45,196 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:46,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:18:46,881 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:18:47,197 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:49,198 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:51,199 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:53,200 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:54,829 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:18:55,201 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:57,202 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:18:59,203 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:01,204 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:02,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:19:02,169 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:19:03,205 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:05,206 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:07,207 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:09,207 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:11,209 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:13,209 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:15,210 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:17,211 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:17,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:19:17,392 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:19:19,212 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:21,213 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:23,214 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:24,909 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:19:25,214 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:27,215 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:29,216 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:31,217 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:32,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:19:32,734 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:19:33,218 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:35,219 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:37,220 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:39,221 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:41,221 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:43,222 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:45,223 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:47,224 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:47,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:19:47,958 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:19:49,225 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:51,226 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:54,227 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:54,989 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:19:56,228 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:19:58,229 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:00,230 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:02,231 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:03,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:20:03,474 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:20:04,232 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:06,233 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:08,234 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:10,234 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:12,235 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:14,236 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:16,237 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:18,238 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:18,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:20:18,693 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:20:20,238 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:22,239 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:24,240 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:25,069 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:20:26,241 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:28,241 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:30,242 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:32,243 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:33,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:20:33,987 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:20:34,244 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:36,245 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:38,246 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:40,247 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:43,248 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:45,249 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:47,250 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:49,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:20:49,212 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:20:49,251 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:51,252 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:53,253 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:55,149 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:20:55,254 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:57,255 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:20:59,256 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:01,257 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:03,258 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:04,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:21:04,436 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:21:05,259 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:07,260 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:09,261 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:11,262 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:13,263 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:15,264 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:17,264 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:19,265 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:19,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:21:19,745 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:21:21,266 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:23,267 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:25,229 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:21:25,268 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:27,269 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:29,270 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:31,271 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:33,272 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:34,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:21:34,971 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:21:35,273 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:37,274 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:39,275 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:41,276 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:43,277 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:45,278 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:47,279 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:49,280 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:50,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:21:50,243 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:21:51,281 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:53,282 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:55,283 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:21:55,309 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:21:57,284 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:00,286 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:02,287 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:04,288 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:05,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:22:05,470 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:22:06,289 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:08,290 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:10,291 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:12,292 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:14,293 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:16,294 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:18,295 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:20,296 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:20,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:22:20,766 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:22:22,297 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:24,298 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:25,390 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:22:26,298 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:28,299 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:30,300 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:32,301 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:34,302 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:36,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:22:36,251 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:22:36,303 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:38,304 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:40,305 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:42,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:44,308 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:46,309 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:48,310 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:50,310 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:51,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:22:51,545 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:22:52,312 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:54,312 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:55,471 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:22:56,313 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:22:58,314 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:00,315 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:02,316 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:05,318 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:06,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:23:06,777 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:23:07,319 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:09,320 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:11,321 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:13,322 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:15,323 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:17,324 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:19,325 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:21,326 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:22,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:23:22,269 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:23:23,327 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:25,328 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:25,551 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:23:27,329 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:29,330 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:31,331 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:33,332 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:35,332 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:37,333 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:37,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:23:37,590 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:23:39,334 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:41,335 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:43,336 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:45,337 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:47,338 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:49,339 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:51,340 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:52,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:23:52,840 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:23:53,341 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:55,343 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:55,632 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:23:57,343 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:23:59,344 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:01,345 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:03,346 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:06,348 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:08,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:24:08,131 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:24:08,349 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:10,349 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:12,350 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:14,351 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:16,352 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:18,353 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:20,354 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:22,355 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:23,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:24:23,404 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:24:24,356 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:25,724 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:24:26,357 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:28,358 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:30,359 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:32,359 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:34,360 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:36,361 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:38,362 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:38,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:24:38,629 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:24:40,363 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:42,363 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:44,364 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:46,365 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:48,366 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:50,367 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:52,368 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:53,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:24:53,913 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:24:54,369 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:55,792 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:24:56,369 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:24:58,370 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:00,371 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:02,372 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:04,373 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:06,373 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:08,374 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:09,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:25:09,194 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:25:10,375 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:12,376 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:14,377 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:16,378 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:18,378 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:21,380 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:23,381 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:24,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:25:24,418 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:25:25,381 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:25,872 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:25:27,382 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:29,383 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:31,384 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:33,385 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:35,386 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:37,387 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:39,388 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:39,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:25:39,786 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:25:41,388 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:43,389 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:45,390 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:47,391 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:49,392 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:51,393 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:53,393 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:55,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:25:55,033 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:25:55,394 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:55,951 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:25:57,395 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:25:59,396 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:01,397 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:03,398 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:05,399 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:07,399 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:09,400 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:10,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:26:10,305 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:26:11,401 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:13,402 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:15,403 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:17,403 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:19,404 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:21,405 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:23,406 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:25,407 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:25,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:26:25,638 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:26:26,032 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:26:27,408 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:29,409 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:31,410 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:33,411 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:35,411 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:38,413 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:40,414 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:40,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:26:40,859 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:26:42,415 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:44,415 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:46,416 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:48,417 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:50,418 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:52,419 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:54,420 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:56,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:26:56,082 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:26:56,300 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:26:56,421 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:26:58,422 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:00,422 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:02,423 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:04,424 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:06,425 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:08,426 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:10,427 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:11,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:27:11,370 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:27:12,428 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:14,428 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:16,429 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:18,430 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:20,431 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:22,432 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:24,433 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:26,191 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:27:26,434 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:26,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:27:26,666 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:27:28,434 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:30,435 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:32,436 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:34,437 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:36,438 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:38,439 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:40,440 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:42,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:27:42,026 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:27:42,441 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:44,442 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:46,442 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:48,443 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:50,444 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:52,445 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:54,446 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:56,271 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:27:56,447 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:27:57,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:27:57,316 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:27:58,448 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:00,449 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:02,450 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:04,451 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:06,452 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:08,453 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:10,454 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:12,454 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:12,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:28:12,578 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:28:14,455 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:16,456 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:18,457 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:20,458 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:22,459 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:24,460 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:26,350 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:28:26,461 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:27,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:28:27,797 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:28:28,462 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:30,462 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:32,463 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:34,464 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:37,466 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:39,466 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:41,467 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:43,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:28:43,046 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:28:43,468 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:45,469 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:47,470 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:49,471 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:51,471 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:53,472 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:55,473 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:56,429 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:28:57,474 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:28:58,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:28:58,270 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:28:59,475 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:01,476 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:03,477 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:05,478 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:07,478 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:09,479 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:11,480 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:13,481 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:13,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:29:13,589 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:29:15,482 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:17,483 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:19,484 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:21,485 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:23,486 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:25,486 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:26,509 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:29:27,487 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:28,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:29:28,835 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:29:30,489 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:32,489 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:34,490 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:36,491 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:38,492 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:40,493 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:42,494 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:44,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:29:44,060 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:29:44,495 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:46,495 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:48,496 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:50,497 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:52,498 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:54,499 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:56,500 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:56,590 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:29:58,501 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:29:59,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:29:59,277 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:30:00,502 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:02,503 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:04,504 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:06,505 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:08,505 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:10,506 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:12,507 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:14,508 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:14,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:30:14,654 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:30:16,509 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:18,510 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:20,510 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:22,511 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:24,512 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:26,513 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:26,667 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:30:28,514 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:29,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:30:29,880 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:30:30,515 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:32,516 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:34,516 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:36,517 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:38,518 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:40,519 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:42,520 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:44,521 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:45,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:30:45,170 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:30:46,522 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:48,523 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:50,524 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:52,524 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:54,525 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:56,526 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:30:56,747 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:30:58,527 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:00,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:31:00,539 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:31:01,528 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:03,529 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:05,530 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:07,531 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:09,531 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:11,532 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:13,533 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:15,534 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:15,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:31:15,846 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:31:17,535 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:19,535 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:21,536 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:23,537 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:25,538 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:26,826 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:31:27,539 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:29,539 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:31,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:31:31,168 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:31:31,540 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:33,541 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:35,542 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:37,542 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:39,543 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:41,544 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:43,545 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:45,546 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:46,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:31:46,397 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:31:47,547 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:49,547 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:51,548 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:53,549 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:55,550 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:31:56,906 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:31:57,551 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:00,552 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:01,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:32:01,711 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:32:02,553 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:04,554 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:06,554 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:08,555 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:10,556 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:12,557 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:14,558 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:16,559 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:16,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:32:16,934 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:32:18,560 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:20,561 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:22,562 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:24,562 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:26,563 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:26,985 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:32:28,564 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:30,565 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:32,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:32:32,235 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:32:32,566 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:34,566 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:36,567 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:38,568 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:40,569 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:42,570 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:44,571 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:46,572 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:47,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:32:47,536 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:32:48,573 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:50,573 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:52,574 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:54,575 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:56,576 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:32:57,067 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:32:58,577 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:00,578 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:02,579 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:02,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:33:02,819 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:33:04,580 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:06,580 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:08,581 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:10,582 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:12,583 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:14,583 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:16,584 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:18,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:33:18,059 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:33:18,585 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:21,586 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:23,587 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:25,588 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:27,146 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:33:27,589 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:29,589 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:31,590 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:33,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:33:33,324 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:33:33,591 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:35,592 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:37,592 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:39,593 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:41,594 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:43,595 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:45,595 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:47,596 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:48,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:33:48,547 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:33:49,597 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:51,598 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:53,598 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:55,599 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:57,226 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:33:57,600 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:33:59,601 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:01,602 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:03,602 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:03,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:34:03,918 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:34:05,603 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:07,604 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:09,605 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:11,605 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:13,606 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:15,607 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:17,608 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:19,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:34:19,155 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:34:19,608 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:21,609 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:23,610 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:25,611 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:27,306 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:34:27,611 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:29,612 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:31,614 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:33,614 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:34,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:34:34,373 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:34:35,615 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:37,616 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:39,617 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:41,617 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:43,618 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:45,619 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:48,620 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:49,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:34:49,687 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:34:50,621 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:52,621 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:54,622 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:56,623 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:34:57,388 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:34:58,624 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:00,624 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:02,625 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:04,626 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:04,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:35:04,915 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:35:06,627 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:08,628 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:10,629 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:12,629 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:14,630 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:16,631 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:18,632 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:20,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:35:20,191 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:35:20,633 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:22,633 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:24,634 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:26,635 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:27,467 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:35:28,636 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:30,636 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:32,637 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:34,638 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:35,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:35:35,418 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:35:36,640 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:38,641 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:41,643 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:43,643 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:45,644 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:47,645 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:49,646 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:50,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:35:50,641 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:35:51,647 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:53,648 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:55,649 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:57,546 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:35:57,649 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:35:59,650 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:01,651 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:03,652 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:05,652 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:05,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:36:05,866 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:36:07,653 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:09,654 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:11,655 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:13,656 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:15,657 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:17,658 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:19,658 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:21,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:36:21,239 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:36:21,659 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:23,660 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:25,661 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:27,627 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:36:27,662 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:29,663 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:31,664 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:33,664 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:35,665 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:36,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:36:36,541 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:36:37,666 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:39,667 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:41,668 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:43,669 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:45,669 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:47,670 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:49,671 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:51,672 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:51,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:36:51,766 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:36:53,673 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:55,673 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:57,674 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:36:57,707 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:36:59,675 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:01,676 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:03,677 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:05,678 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:07,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:37:07,108 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:37:08,679 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:10,680 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:12,681 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:14,682 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:16,683 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:18,684 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:20,685 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:22,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:37:22,333 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:37:22,685 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:24,686 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:26,687 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:27,786 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:37:28,688 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:30,689 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:32,690 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:34,691 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:36,691 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:37,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:37:37,597 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:37:38,692 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:40,693 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:42,694 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:44,695 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:46,695 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:48,696 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:50,697 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:52,697 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:52,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:37:52,833 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:37:54,698 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:56,699 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:37:57,865 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:37:58,700 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:00,701 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:02,702 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:04,702 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:06,703 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:08,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:38:08,069 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:38:08,704 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:10,705 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:12,706 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:15,707 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:17,708 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:19,708 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:21,709 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:23,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:38:23,612 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:38:23,710 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:25,710 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:27,711 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:27,944 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:38:29,712 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:31,713 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:33,713 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:35,714 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:37,715 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:38,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:38:38,873 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:38:39,716 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:41,716 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:43,717 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:45,718 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:47,719 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:49,720 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:51,720 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:53,721 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:54,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:38:54,164 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:38:55,722 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:57,723 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:38:58,023 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:38:59,723 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:01,724 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:03,725 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:05,726 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:07,726 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:09,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:39:09,424 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:39:09,727 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:11,728 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:13,729 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:15,729 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:17,730 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:19,731 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:21,732 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:23,732 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:24,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:39:24,713 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:39:25,734 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:28,102 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:39:28,735 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:30,736 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:32,736 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:34,737 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:36,738 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:38,739 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:40,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:39:40,071 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:39:40,739 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:42,740 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:44,741 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:46,741 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:48,742 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:50,743 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:52,744 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:54,744 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:55,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:39:55,371 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:39:56,745 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:39:58,186 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:39:58,746 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:00,746 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:02,747 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:04,748 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:06,749 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:08,749 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:10,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:40:10,593 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:40:10,750 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:12,751 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:14,751 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:16,752 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:18,753 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:20,754 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:22,754 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:24,755 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:25,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:40:25,822 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:40:26,756 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:28,264 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:40:28,757 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:30,757 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:32,758 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:34,759 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:36,759 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:38,760 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:40,761 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:41,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:40:41,118 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:40:43,762 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:45,763 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:47,764 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:49,764 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:51,765 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:53,766 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:55,767 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:56,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:40:56,520 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:40:57,767 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:40:58,343 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:40:59,768 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:01,769 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:03,770 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:05,770 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:07,771 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:09,772 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:11,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:41:11,742 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:41:11,773 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:13,774 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:15,775 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:17,776 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:19,777 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:21,778 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:23,779 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:25,780 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:27,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:41:27,071 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:41:27,781 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:28,421 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:41:29,782 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:31,783 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:33,784 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:35,785 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:37,786 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:39,787 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:41,788 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:42,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:41:42,310 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:41:43,789 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:45,790 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:48,792 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:50,793 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:52,794 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:54,795 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:56,796 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:41:57,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:41:57,533 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:41:58,500 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:41:58,797 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:00,798 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:02,799 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:04,800 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:06,801 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:08,802 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:10,804 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:12,805 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:12,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:42:12,923 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:42:14,806 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:16,807 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:18,808 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:20,809 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:22,810 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:24,811 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:26,812 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:28,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:42:28,286 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:42:28,577 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:42:28,813 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:30,814 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:32,815 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:34,816 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:36,817 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:38,818 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:40,819 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:42,819 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:43,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:42:43,616 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:42:44,821 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:46,822 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:48,823 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:50,824 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:52,825 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:54,826 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:56,826 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:58,656 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:42:58,827 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:42:58,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:42:58,853 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:43:01,828 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:03,829 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:05,830 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:07,831 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:09,832 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:11,832 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:13,833 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:14,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:43:14,228 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:43:15,834 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:17,835 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:19,836 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:21,837 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:23,838 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:25,838 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:27,839 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:28,736 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:43:29,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:43:29,450 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:43:29,840 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:31,841 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:33,842 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:35,843 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:37,844 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:39,845 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:41,846 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:43,847 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:44,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:43:44,675 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:43:45,848 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:47,849 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:49,850 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:51,850 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:53,851 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:55,852 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:57,853 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:58,820 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:43:59,854 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:43:59,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:43:59,968 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:44:01,855 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:03,856 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:05,856 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:07,857 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:09,858 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:12,860 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:14,861 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:15,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:44:15,351 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:44:16,862 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:18,863 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:20,864 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:22,865 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:24,866 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:26,867 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:28,868 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:28,899 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:44:30,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:44:30,684 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:44:30,869 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:32,869 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:34,870 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:36,871 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:38,871 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:40,872 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:42,873 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:44,873 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:45,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:44:45,909 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:44:46,874 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:48,875 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:50,875 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:52,876 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:54,877 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:56,877 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:58,878 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:44:58,975 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:45:00,878 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:01,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:45:01,228 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:45:02,879 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:04,880 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:06,880 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:08,881 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:10,882 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:12,882 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:14,883 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:16,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:45:16,454 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:45:16,884 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:18,885 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:20,886 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:23,887 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:25,888 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:27,889 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:29,051 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:45:29,890 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:31,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:45:31,761 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:45:31,890 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:33,891 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:35,892 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:37,893 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:39,894 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:41,895 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:43,896 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:45,896 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:47,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:45:47,033 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:45:47,897 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:49,898 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:51,899 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:53,900 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:55,901 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:57,902 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:45:59,130 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:45:59,903 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:01,904 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:02,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:46:02,257 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:46:03,905 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:05,906 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:07,906 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:09,907 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:11,908 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:13,909 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:15,910 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:17,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:46:17,553 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:46:17,911 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:19,912 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:21,913 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:23,914 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:25,915 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:27,915 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:29,210 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:46:29,916 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:31,917 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:32,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:46:32,803 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:46:33,918 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:35,919 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:37,920 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:39,921 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:41,922 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:43,922 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:45,923 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:47,924 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:48,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:46:48,044 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:46:50,926 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:52,927 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:54,928 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:56,928 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:58,929 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:46:59,287 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:47:00,930 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:02,931 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:03,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:47:03,358 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:47:04,932 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:06,933 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:08,934 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:10,935 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:12,935 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:14,936 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:16,937 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:18,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:47:18,608 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:47:18,938 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:20,939 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:22,940 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:24,941 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:26,942 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:28,942 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:29,371 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:47:30,943 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:32,944 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:33,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:47:33,860 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:47:34,945 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:36,946 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:38,947 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:40,948 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:42,948 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:44,949 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:46,950 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:48,951 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:49,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:47:49,134 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:47:50,952 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:52,953 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:54,954 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:56,955 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:58,955 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:47:59,450 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:48:00,956 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:02,957 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:04,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:48:04,408 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:48:04,958 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:06,959 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:08,960 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:10,960 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:12,961 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:14,962 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:16,963 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:18,964 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:19,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:48:19,698 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:48:20,965 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:22,966 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:24,967 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:27,968 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:29,530 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:48:29,969 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:31,970 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:33,971 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:34,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:48:34,988 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:48:35,972 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:37,973 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:39,973 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:41,974 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:43,975 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:45,976 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:47,977 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:49,978 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:50,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:48:50,211 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:48:51,979 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:53,980 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:55,980 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:57,981 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:48:59,607 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:48:59,982 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:01,983 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:03,984 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:05,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:49:05,561 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:49:05,985 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:07,986 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:09,986 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:11,987 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:13,988 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:15,989 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:17,990 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:19,990 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:20,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:49:20,866 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:49:21,991 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:23,992 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:25,993 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:27,993 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:29,687 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:49:29,994 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:31,995 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:34,996 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:36,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:49:36,086 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:49:36,997 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:38,998 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:40,999 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:43,000 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:45,000 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:47,001 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:49,002 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:51,003 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:51,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:49:51,412 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:49:53,004 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:55,005 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:57,006 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:59,006 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:49:59,768 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:50:01,007 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:03,008 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:05,009 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:06,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:50:06,645 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:50:07,010 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:09,011 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:11,012 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:13,012 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:15,013 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:17,014 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:19,015 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:21,016 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:21,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:50:21,927 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:50:23,016 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:25,017 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:27,018 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:29,019 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:29,848 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:50:31,020 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:33,020 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:35,021 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:37,022 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:37,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:50:37,288 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:50:39,023 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:41,024 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:43,024 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:45,025 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:47,026 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:49,027 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:51,028 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:52,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:50:52,513 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:50:53,028 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:55,029 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:57,030 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:59,031 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:50:59,928 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:51:01,032 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:03,032 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:05,033 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:07,034 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:07,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:51:07,764 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:51:09,035 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:11,036 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:13,036 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:16,038 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:18,038 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:20,039 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:22,040 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:23,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:51:23,006 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:51:24,041 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:26,042 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:28,042 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:30,007 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:51:30,043 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:32,044 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:34,045 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:36,046 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:38,046 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:38,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:51:38,280 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:51:40,047 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:42,048 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:44,049 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:46,050 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:48,050 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:50,051 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:52,052 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:53,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:51:53,553 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:51:54,053 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:56,054 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:51:58,055 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:00,056 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:00,086 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:52:02,057 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:04,058 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:06,059 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:08,060 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:08,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:52:08,776 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:52:10,061 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:12,062 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:14,063 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:16,064 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:18,065 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:20,066 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:22,067 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:24,068 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:24,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:52:24,150 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:52:26,069 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:28,070 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:30,071 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:30,165 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:52:32,072 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:34,073 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:36,074 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:38,075 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:39,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:52:39,371 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:52:40,076 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:42,077 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:44,078 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:46,079 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:49,080 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:51,082 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:53,083 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:54,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:52:54,597 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:52:55,083 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:57,084 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:52:59,085 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:00,249 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:53:01,086 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:03,087 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:05,088 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:07,089 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:09,090 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:09,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:53:09,822 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:53:11,091 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:13,092 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:15,093 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:17,094 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:19,095 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:21,096 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:23,097 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:25,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:53:25,089 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:53:25,098 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:27,099 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:29,100 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:30,329 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:53:31,101 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:33,102 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:35,103 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:37,104 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:39,105 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:40,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:53:40,413 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:53:41,106 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:43,107 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:45,108 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:47,109 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:50,111 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:52,112 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:54,113 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:55,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:53:55,700 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:53:56,113 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:53:58,114 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:00,115 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:00,408 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:54:02,117 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:04,118 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:06,119 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:08,120 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:10,121 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:10,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:54:10,925 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:54:12,122 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:14,123 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:16,124 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:18,125 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:20,126 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:22,127 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:24,128 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:26,129 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:26,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:54:26,242 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:54:28,130 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:30,131 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:30,488 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:54:32,132 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:34,133 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:36,134 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:38,135 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:40,136 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:41,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:54:41,538 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:54:42,137 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:44,138 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:46,139 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:48,140 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:50,141 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:52,142 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:54,143 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:56,144 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:54:56,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:54:56,761 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:54:58,145 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:00,146 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:00,567 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:55:02,147 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:04,148 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:06,149 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:08,150 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:10,151 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:12,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:55:12,087 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:55:12,152 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:14,153 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:16,154 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:18,155 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:20,156 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:22,157 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:24,158 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:26,159 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:27,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:55:27,407 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:55:28,160 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:30,647 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:55:31,161 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:33,162 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:35,163 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:37,165 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:39,166 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:41,167 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:42,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:55:42,631 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:55:43,168 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:45,169 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:47,170 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:49,171 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:51,172 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:53,173 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:55,174 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:57,175 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:55:57,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:55:57,936 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:55:59,176 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:00,725 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:56:01,177 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:03,178 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:05,178 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:07,179 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:09,180 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:11,181 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:13,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:56:13,158 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:56:13,182 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:15,182 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:17,183 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:19,184 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:21,185 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:23,186 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:25,186 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:27,187 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:28,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:56:28,491 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:56:29,188 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:30,805 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:56:31,189 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:33,190 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:35,190 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:37,191 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:39,192 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:41,192 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:43,193 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:43,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:56:43,813 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:56:46,194 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:48,195 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:50,196 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:52,197 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:54,198 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:56,199 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:58,200 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:56:59,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:56:59,037 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:57:00,201 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:00,886 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:57:02,202 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:04,203 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:06,204 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:08,204 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:10,205 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:12,206 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:14,207 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:14,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:57:14,394 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:57:16,208 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:18,209 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:20,210 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:22,210 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:24,211 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:26,212 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:28,213 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:29,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:57:29,660 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:57:30,214 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:30,966 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:57:32,214 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:34,215 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:36,216 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:38,217 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:40,218 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:42,218 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:44,219 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:44,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:57:44,968 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:57:46,222 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:48,222 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:50,223 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:52,224 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:54,225 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:56,225 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:57:58,226 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:00,227 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:00,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:58:00,306 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:58:01,046 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:58:02,228 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:04,228 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:06,229 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:09,230 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:11,231 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:13,232 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:15,232 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:15,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:58:15,590 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:58:17,233 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:19,234 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:21,235 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:23,235 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:25,236 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:27,237 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:29,238 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:30,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:58:30,839 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:58:31,126 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:58:31,238 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:33,239 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:35,240 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:37,241 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:39,242 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:41,242 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:43,243 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:45,244 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:46,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:58:46,083 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:58:47,245 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:49,245 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:51,246 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:53,247 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:55,248 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:57,249 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:58:59,249 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:01,206 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:59:01,250 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:01,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:59:01,384 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:59:03,251 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:05,252 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:07,253 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:09,254 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:11,254 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:13,255 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:15,256 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:16,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:59:16,729 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:59:17,257 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:19,258 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:21,258 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:23,259 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:25,260 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:27,261 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:29,262 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:31,262 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:31,287 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 12:59:32,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:59:32,024 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:59:33,263 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:35,264 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:37,264 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:39,265 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:41,266 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:43,267 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:45,267 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:47,268 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:47,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 12:59:47,301 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 12:59:49,269 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:51,270 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:54,271 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:56,272 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 12:59:58,272 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:00,273 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:01,367 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:00:02,274 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:03,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:00:03,102 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:00:04,275 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:06,275 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:08,276 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:10,277 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:12,278 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:14,278 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:16,279 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:18,280 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:18,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:00:18,534 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:00:20,280 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:22,281 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:24,282 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:26,283 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:28,283 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:30,284 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:31,447 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:00:32,285 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:33,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:00:33,760 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:00:34,285 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:36,286 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:38,287 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:40,288 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:42,288 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:44,289 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:46,290 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:48,290 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:49,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:00:49,071 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:00:50,291 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:52,292 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:54,293 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:56,294 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:00:58,294 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:00,295 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:01,527 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:01:02,296 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:04,296 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:04,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:01:04,418 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:01:06,297 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:08,298 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:10,298 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:12,299 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:14,300 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:16,301 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:18,301 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:19,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:01:19,730 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:01:20,302 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:23,304 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:25,304 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:27,305 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:29,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:31,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:31,607 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:01:33,307 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:34,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:01:34,952 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:01:35,308 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:37,309 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:39,309 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:41,310 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:43,311 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:45,312 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:47,312 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:49,313 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:50,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:01:50,173 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:01:51,314 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:53,315 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:55,316 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:57,316 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:01:59,317 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:01,318 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:01,686 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:02:03,319 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:05,319 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:05,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:02:05,532 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:02:07,320 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:09,321 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:11,322 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:13,322 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:15,323 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:17,324 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:19,325 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:20,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:02:20,756 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:02:21,326 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:23,327 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:25,327 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:27,328 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:29,329 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:31,330 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:31,767 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:02:33,331 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:35,331 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:36,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:02:36,021 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:02:37,332 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:39,333 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:42,334 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:44,335 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:46,336 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:48,337 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:50,338 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:51,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:02:51,327 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:02:52,339 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:54,340 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:56,341 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:02:58,342 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:00,343 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:01,846 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:03:02,344 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:04,344 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:06,345 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:06,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:03:06,751 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:03:08,346 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:10,347 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:12,348 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:14,349 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:16,350 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:18,351 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:20,352 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:21,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:03:21,975 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:03:22,352 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:24,353 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:26,354 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:28,355 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:30,356 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:31,926 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:03:32,356 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:34,357 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:36,358 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:37,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:03:37,196 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:03:38,359 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:40,360 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:42,361 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:44,361 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:46,362 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:48,363 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:51,364 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:52,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:03:52,553 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:03:53,365 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:55,366 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:57,367 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:03:59,368 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:01,369 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:02,004 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:04:03,369 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:05,370 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:07,371 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:07,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:04:07,772 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:04:09,372 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:11,373 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:13,374 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:15,375 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:17,375 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:19,376 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:21,377 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:23,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:04:23,120 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:04:23,378 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:25,379 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:27,380 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:29,381 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:31,382 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:32,084 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:04:33,383 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:35,384 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:37,385 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:38,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:04:38,406 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:04:40,386 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:42,386 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:44,387 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:46,388 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:48,389 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:50,390 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:52,391 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:53,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:04:53,735 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:04:54,392 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:56,393 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:04:58,394 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:00,395 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:02,164 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:05:02,395 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:04,396 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:06,397 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:08,398 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:08,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:05:08,971 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:05:10,399 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:12,400 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:14,401 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:16,402 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:18,403 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:20,403 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:22,404 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:24,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:05:24,394 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:05:24,405 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:26,406 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:28,407 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:30,408 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:32,243 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:05:32,409 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:34,409 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:36,410 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:38,411 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:39,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:05:39,689 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:05:40,412 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:42,413 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:44,414 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:47,415 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:49,416 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:51,417 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:53,418 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:54,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:05:54,914 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:05:55,419 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:57,420 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:05:59,421 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:01,422 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:02,323 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:06:03,423 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:05,424 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:07,425 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:09,426 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:10,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:06:10,135 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:06:11,426 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:13,427 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:15,429 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:17,430 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:19,431 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:21,432 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:23,433 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:25,433 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:25,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:06:25,502 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:06:27,435 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:29,436 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:31,437 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:32,404 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:06:33,438 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:35,439 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:37,440 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:39,441 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:40,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:06:40,877 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:06:41,442 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:43,443 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:45,444 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:47,445 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:49,446 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:51,447 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:53,448 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:55,449 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:56,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:06:56,106 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:06:57,450 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:06:59,451 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:01,452 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:02,484 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:07:03,453 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:05,454 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:07,455 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:09,457 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:11,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:07:11,333 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:07:11,457 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:13,458 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:15,459 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:18,460 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:20,461 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:22,462 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:24,464 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:26,465 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:26,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:07:26,686 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:07:28,466 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:30,467 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:32,469 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:32,565 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:07:34,469 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:36,470 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:38,471 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:40,472 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:41,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:07:41,912 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:07:42,473 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:44,474 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:46,475 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:48,476 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:50,477 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:52,478 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:54,479 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:56,480 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:07:57,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:07:57,325 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:07:58,481 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:00,482 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:02,483 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:02,645 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:08:04,484 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:06,485 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:08,486 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:10,487 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:12,488 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:12,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:08:12,548 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:08:14,489 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:16,490 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:19,491 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:21,492 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:23,493 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:25,494 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:27,495 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:27,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:08:27,762 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:08:29,496 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:31,497 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:32,725 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:08:33,498 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:35,498 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:37,499 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:39,500 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:41,501 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:43,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:08:43,079 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:08:43,502 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:45,503 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:47,504 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:49,505 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:51,506 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:53,507 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:55,507 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:57,508 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:08:58,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:08:58,343 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:08:59,509 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:01,510 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:02,806 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:09:03,511 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:05,512 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:07,513 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:09,514 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:11,515 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:13,516 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:13,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:09:13,567 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:09:15,517 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:17,517 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:19,518 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:21,519 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:24,520 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:26,521 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:28,522 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:28,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:09:28,864 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:09:30,523 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:32,524 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:32,886 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:09:34,525 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:36,526 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:38,527 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:40,528 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:42,529 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:44,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:09:44,297 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:09:44,530 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:46,531 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:48,532 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:50,534 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:52,534 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:54,536 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:56,537 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:58,538 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:09:59,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:09:59,588 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:10:00,539 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:02,540 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:02,967 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:10:04,541 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:06,542 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:08,543 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:10,544 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:12,545 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:14,545 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:14,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:10:14,810 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:10:16,546 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:18,547 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:20,548 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:22,549 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:24,549 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:26,550 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:28,551 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:30,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:10:30,107 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:10:30,552 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:32,553 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:33,048 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:10:34,554 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:36,555 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:38,556 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:40,557 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:42,558 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:44,559 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:45,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:10:45,431 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:10:46,560 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:48,561 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:50,562 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:52,563 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:54,564 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:56,565 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:10:58,566 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:00,567 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:00,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:11:00,656 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:11:02,568 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:03,128 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:11:04,569 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:06,569 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:08,570 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:10,571 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:12,572 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:15,574 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:16,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:11:16,013 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:11:17,575 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:19,576 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:21,577 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:23,578 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:25,579 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:27,580 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:29,581 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:31,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:11:31,308 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:11:31,582 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:33,208 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:11:33,583 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:35,584 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:37,585 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:39,586 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:41,587 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:43,588 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:45,589 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:46,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:11:46,618 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:11:47,590 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:49,591 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:51,592 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:53,593 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:55,594 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:57,595 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:11:59,596 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:01,597 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:01,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:12:01,847 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:12:03,287 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:12:03,598 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:05,599 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:07,600 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:10,601 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:12,602 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:14,603 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:16,604 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:17,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:12:17,151 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:12:18,605 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:20,606 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:22,607 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:24,608 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:26,609 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:28,610 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:30,611 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:32,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:12:32,438 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:12:32,612 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:33,370 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:12:34,613 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:36,614 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:38,615 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:40,616 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:42,617 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:44,618 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:46,619 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:47,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:12:47,737 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:12:48,620 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:50,621 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:52,622 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:54,623 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:56,624 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:12:58,625 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:00,626 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:02,627 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:03,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:13:03,041 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:13:03,449 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:13:04,628 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:06,629 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:08,630 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:10,631 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:12,632 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:15,633 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:17,634 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:18,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:13:18,347 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:13:19,634 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:21,635 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:23,636 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:25,637 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:27,637 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:29,638 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:31,639 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:33,529 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:13:33,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:13:33,572 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:13:33,640 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:35,640 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:37,641 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:39,642 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:41,643 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:43,643 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:45,644 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:47,645 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:48,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:13:48,796 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:13:49,646 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:51,647 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:53,648 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:55,648 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:57,649 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:13:59,650 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:01,651 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:03,605 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:14:03,652 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:04,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:14:04,098 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:14:05,652 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:07,653 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:09,654 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:11,655 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:13,656 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:16,657 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:18,658 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:19,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:14:19,346 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:14:20,659 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:22,659 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:24,660 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:26,661 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:28,662 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:30,663 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:32,664 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:33,682 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:14:34,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:14:34,569 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:14:34,664 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:36,665 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:38,666 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:40,667 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:42,668 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:44,669 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:46,670 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:48,671 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:49,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:14:49,907 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:14:50,671 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:52,672 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:54,673 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:56,674 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:14:58,674 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:00,675 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:02,676 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:03,757 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:15:04,677 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:05,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:15:05,130 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:15:06,678 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:08,678 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:10,679 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:12,680 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:14,681 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:16,682 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:18,683 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:20,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:15:20,421 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:15:20,684 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:22,685 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:24,686 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:26,686 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:28,687 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:30,688 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:33,689 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:33,832 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:15:35,690 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:35,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:15:35,763 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:15:37,691 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:39,692 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:41,693 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:43,694 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:45,694 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:47,695 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:49,696 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:50,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:15:50,985 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:15:51,697 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:53,698 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:55,698 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:57,699 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:15:59,700 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:01,701 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:03,701 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:03,909 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:16:05,702 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:06,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:16:06,240 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:16:07,703 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:09,704 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:11,705 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:13,706 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:15,707 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:17,709 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:19,710 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:21,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:16:21,535 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:16:21,711 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:23,712 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:25,712 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:27,713 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:29,714 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:31,715 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:33,716 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:33,984 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:16:35,717 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:36,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:16:36,901 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:16:37,718 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:39,718 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:41,719 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:43,720 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:45,721 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:47,721 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:49,722 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:51,723 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:52,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:16:52,128 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:16:53,724 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:55,725 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:57,726 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:16:59,727 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:01,728 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:03,729 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:04,061 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:17:05,730 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:07,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:17:07,417 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:17:07,731 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:09,732 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:11,732 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:13,733 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:15,734 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:17,735 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:19,736 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:21,737 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:22,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:17:22,690 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:17:23,738 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:25,738 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:27,739 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:29,740 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:31,741 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:34,139 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:17:34,742 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:36,743 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:37,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:17:37,991 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:17:38,744 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:40,745 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:42,746 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:44,747 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:46,747 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:48,748 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:50,749 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:52,750 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:53,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:17:53,215 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:17:54,751 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:56,752 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:17:58,753 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:00,753 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:02,754 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:04,217 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:18:04,755 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:06,756 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:08,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:18:08,468 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:18:08,757 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:10,757 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:12,759 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:14,759 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:16,760 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:18,761 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:20,762 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:22,763 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:23,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:18:23,696 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:18:24,764 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:26,764 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:28,765 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:30,766 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:32,767 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:34,296 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:18:34,768 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:36,769 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:38,770 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:39,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:18:39,028 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:18:40,771 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:42,772 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:44,773 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:46,774 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:48,775 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:50,776 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:52,776 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:54,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:18:54,420 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:18:54,777 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:56,778 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:18:58,779 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:00,780 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:02,781 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:04,373 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:19:04,782 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:06,783 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:08,784 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:09,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:19:09,788 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:19:10,785 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:13,787 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:15,788 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:17,789 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:19,790 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:21,791 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:23,792 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:25,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:19:25,118 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:19:25,793 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:27,794 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:29,795 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:31,796 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:33,797 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:34,451 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:19:35,798 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:37,799 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:39,801 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:40,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:19:40,348 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:19:41,802 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:43,803 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:45,803 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:47,805 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:49,806 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:51,807 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:53,808 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:55,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:19:55,595 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:19:55,809 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:57,810 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:19:59,811 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:01,812 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:03,813 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:04,531 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:20:05,814 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:07,815 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:09,815 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:10,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:20:10,839 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:20:11,816 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:13,817 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:15,818 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:17,819 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:19,820 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:21,821 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:23,821 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:25,822 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:26,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:20:26,770 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:20:27,823 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:29,824 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:31,825 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:33,826 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:34,616 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:20:35,826 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:37,827 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:39,828 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:41,829 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:41,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:20:41,995 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:20:43,830 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:45,830 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:47,831 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:50,832 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:52,833 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:54,834 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:56,835 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:20:57,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:20:57,283 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:20:58,836 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:00,836 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:02,837 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:04,692 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:21:04,838 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:06,839 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:08,840 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:10,841 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:12,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:21:12,516 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:21:12,841 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:14,842 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:16,843 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:18,844 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:20,845 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:22,845 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:24,846 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:26,847 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:27,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:21:27,926 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:21:28,848 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:30,849 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:32,850 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:34,770 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:21:34,851 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:36,852 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:38,853 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:40,854 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:42,855 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:43,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:21:43,150 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:21:44,856 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:46,856 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:48,857 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:50,858 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:52,859 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:54,860 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:56,860 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:21:58,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:21:58,376 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:21:58,861 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:00,862 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:02,863 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:04,847 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:22:04,863 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:06,864 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:09,865 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:11,866 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:13,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:22:13,664 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:22:13,868 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:15,868 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:17,869 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:19,870 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:21,870 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:23,871 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:25,872 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:27,873 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:29,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:22:29,013 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:22:29,874 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:31,875 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:33,875 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:34,929 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:22:35,876 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:37,877 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:39,878 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:41,879 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:43,880 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:44,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:22:44,318 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:22:45,880 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:47,882 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:49,883 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:51,884 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:53,884 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:55,885 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:57,886 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:22:59,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:22:59,611 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:22:59,887 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:23:05,008 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:23:14,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:23:14,851 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:23:30,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:23:30,098 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:23:35,087 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:23:45,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:23:45,369 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:24:00,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:24:00,630 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:24:05,166 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:24:12,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:24:13,004 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 13:24:13,060 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 13:24:13,131 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 13:24:13,915 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:24:13,915 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 13:24:14,915 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:24:15,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:24:15,882 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:24:15,916 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:24:19,918 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:24:31,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:24:31,108 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:24:35,251 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:24:35,925 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:24:46,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:24:46,333 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:25:01,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:25:01,556 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:25:05,335 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:25:16,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:25:16,784 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:25:32,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:25:32,011 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:25:35,419 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:25:47,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:25:47,239 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:26:02,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:26:02,463 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:26:05,501 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:26:17,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:26:17,692 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:26:32,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:26:32,917 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:26:35,580 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:26:48,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:26:48,142 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:27:03,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:27:03,367 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:27:05,660 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:27:18,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:27:18,595 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:27:33,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:27:33,854 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:27:35,739 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:27:49,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:27:49,322 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:28:04,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:28:04,549 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:28:05,821 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:28:19,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:28:19,777 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:28:35,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:28:35,030 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:28:35,904 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:28:40,045 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:28:42,045 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:28:44,046 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:28:46,047 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:28:48,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:28:48,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:28:48,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:28:48,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:28:48,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:28:48,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:28:48,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:28:48,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:28:48,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:28:48,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:28:48,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:28:48,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:28:48,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:28:48,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:28:48,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:28:48,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:28:49,049 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:28:50,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:28:50,258 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:28:51,050 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:28:53,050 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:28:55,051 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:28:57,052 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:28:59,053 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:29:01,054 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:29:03,055 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:29:05,056 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:29:05,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:29:05,487 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:29:05,985 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:29:07,057 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:29:09,058 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:29:11,059 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:29:13,059 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:29:15,060 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:29:17,061 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:29:19,062 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:29:20,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:29:20,725 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:29:21,063 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:29:22,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:22,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:22,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:22,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:22,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:22,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:22,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:22,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:22,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:22,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:22,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:22,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:22,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:22,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:22,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:22,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:23,064 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:29:25,065 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:29:31,067 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:29:35,069 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:29:35,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:29:35,948 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:29:36,170 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:29:37,070 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:29:41,072 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:29:42,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:42,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:42,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:42,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:42,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:42,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:42,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:42,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:42,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:42,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:42,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:42,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:42,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:42,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:42,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:42,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:45,073 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:29:46,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:46,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:46,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:46,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:46,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:46,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:46,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:46,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:46,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:46,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:46,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:46,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:46,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:46,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:46,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:46,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,075 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:29:49,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:49,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:51,076 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:29:51,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:29:51,374 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:29:52,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:52,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:53,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:53,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:53,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:53,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:53,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:53,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:53,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:53,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:53,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:53,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:53,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:53,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:53,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:53,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:53,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:53,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:54,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,077 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:29:55,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:55,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:57,078 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:29:58,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:58,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:29:59,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:01,080 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:30:01,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:01,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:01,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:01,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:01,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:01,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:01,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:01,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:01,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:01,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:01,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:01,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:01,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:01,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:01,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:01,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:03,081 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:30:04,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:04,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:05,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,144 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:30:06,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:30:06,663 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:30:06,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:06,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:07,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:07,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:07,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:07,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:07,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:07,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:07,084 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:30:07,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:07,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:07,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:07,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:07,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:07,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:07,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:07,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:07,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:07,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,448 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,448 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:08,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:09,085 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:30:10,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:10,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:10,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:10,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:10,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:10,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:10,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:10,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:10,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:10,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:10,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:10,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:10,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:10,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:10,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:10,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:11,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:11,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:11,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:11,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:11,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:11,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:11,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:11,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:11,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:11,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:11,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:11,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:11,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:11,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:11,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:11,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:12,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:13,086 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:30:14,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:14,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:14,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:14,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:14,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:14,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:14,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:14,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:14,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:14,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:14,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:14,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:14,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:14,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:14,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:14,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,087 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:30:15,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:15,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:16,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:17,088 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:30:18,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:18,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:18,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:18,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:18,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:18,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:18,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:18,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:18,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:18,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:18,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:18,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:18,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:18,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:18,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:18,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,090 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:30:20,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:20,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:21,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:21,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:21,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:21,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:21,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:21,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:21,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:21,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:21,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:21,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:21,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:21,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:21,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:21,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:21,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:21,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:21,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:30:21,890 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:30:24,091 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:30:25,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:25,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:25,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:25,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:25,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:25,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:25,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:25,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:25,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:25,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:25,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:25,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:26,092 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:30:28,093 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:30:30,094 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:30:32,095 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:30:34,096 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:30:36,097 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:30:36,224 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:30:37,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:30:37,116 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:30:38,098 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:30:39,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:39,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:39,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:39,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:39,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:39,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:39,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:39,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:39,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:39,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:39,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:39,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:39,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:39,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:39,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:39,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:30:40,099 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:30:42,100 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:30:44,101 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:30:46,102 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:30:48,103 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:30:50,104 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:30:52,105 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:30:52,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:30:52,457 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:30:54,107 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:30:56,107 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:30:58,108 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:31:00,109 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:31:02,110 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:31:04,111 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:31:06,112 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:31:06,304 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:31:07,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:31:07,717 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:31:08,113 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:31:10,114 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:31:12,115 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:31:18,117 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:31:21,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:21,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:21,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:21,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:21,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:21,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:21,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:21,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:21,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:21,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:21,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:21,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:21,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:21,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:21,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:21,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:22,119 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:31:22,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:31:22,987 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:31:26,121 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:31:30,122 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:31:32,123 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:31:33,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:33,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:33,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:33,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:33,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:33,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:33,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:33,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:33,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:33,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:33,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:33,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:33,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:33,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:33,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:33,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:36,126 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:31:36,380 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:31:38,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:31:38,210 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:31:38,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:38,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:38,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:38,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:38,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:38,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:38,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:38,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:38,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:38,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:38,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:38,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:38,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:38,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:38,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:38,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:40,127 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:31:43,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:43,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:43,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:43,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:43,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:43,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:43,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:43,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:43,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:43,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:43,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:43,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:43,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:43,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:43,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:43,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,129 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:31:44,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:44,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:46,130 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:31:47,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:47,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:48,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:49,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,133 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:31:50,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,448 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:50,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,133 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:31:52,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:52,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:53,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:31:53,433 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:31:55,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:55,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:55,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:55,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:55,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:55,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:55,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:55,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:55,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:55,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:55,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:55,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:55,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:55,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:55,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:55,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:56,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:56,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:56,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:56,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:56,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:56,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:56,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:56,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:56,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:56,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:56,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:56,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:56,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:56,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:56,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:56,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:56,135 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:31:57,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:57,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,136 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:31:58,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:58,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:59,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:59,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:59,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:59,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:59,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:59,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:59,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:59,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:59,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:59,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:59,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:59,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:59,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:59,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:59,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:31:59,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:00,137 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:01,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:01,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:01,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:01,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:01,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:01,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:01,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:01,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:01,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:01,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:01,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:01,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:01,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:01,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:01,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:01,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:02,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:02,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:02,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:02,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:02,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:02,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:02,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:02,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:02,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:02,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:02,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:02,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:02,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:02,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:02,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:02,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:03,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:04,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:04,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:04,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:04,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:04,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:04,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:04,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:04,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:04,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:04,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:04,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:04,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:04,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:04,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:04,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:04,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:04,138 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:06,139 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:06,449 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:32:06,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:06,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:06,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:06,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:06,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:06,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:06,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:06,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:06,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:06,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:06,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:06,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:06,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:06,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:06,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:06,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:07,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:07,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:07,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:07,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:07,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:07,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:07,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:07,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:07,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:07,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:07,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:07,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:07,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:07,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:07,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:07,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:07,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:07,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:07,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:07,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:07,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,140 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:08,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:08,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:32:08,690 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:32:09,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:09,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:09,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:09,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:09,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:09,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:09,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:09,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:09,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:09,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:09,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:09,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:09,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:09,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:09,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:09,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:10,141 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:11,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:11,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:11,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:11,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:11,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:11,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:11,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:11,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:11,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:11,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:11,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:11,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:11,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:11,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:11,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:11,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:12,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:12,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:12,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:12,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:12,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:12,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:12,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:12,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:12,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:12,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:12,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:12,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:12,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:12,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:12,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:12,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:13,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:14,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:14,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:14,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:14,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:14,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:14,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:14,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:14,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:14,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:14,580 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 13:32:14,792 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 13:32:14,868 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 13:32:15,143 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:15,143 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 13:32:15,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:15,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:15,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:15,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:15,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:15,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:15,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:15,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:15,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:15,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:15,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:15,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:15,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:15,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:15,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:15,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:16,143 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:17,144 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:19,145 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:21,146 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:23,147 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:23,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:32:23,914 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:32:25,148 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:27,148 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:29,149 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:31,150 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:31,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:31,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:31,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:31,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:31,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:31,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:31,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:31,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:31,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:31,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:31,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:31,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:31,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:31,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:31,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:31,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:32:33,151 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:35,152 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:36,536 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:32:37,153 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:39,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:32:39,137 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:32:39,154 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:41,155 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:43,156 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:45,157 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:47,158 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:49,159 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:51,160 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:53,161 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:54,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:32:54,360 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:32:55,161 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:57,162 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:32:59,163 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:33:01,164 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:33:03,165 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:33:06,622 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:33:09,167 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:33:09,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:33:09,707 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:33:13,169 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:33:17,171 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:33:21,173 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:33:23,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:23,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:23,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:23,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:23,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:23,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:23,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:23,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:23,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:23,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:23,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:23,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:23,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:23,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:23,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:23,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:24,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:33:24,933 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:33:25,174 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:33:27,175 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:33:30,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:30,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:30,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:30,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:30,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:30,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:30,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:30,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:30,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:30,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:30,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:30,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:30,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:30,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:30,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:30,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:31,177 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:33:35,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:35,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:35,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:35,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:35,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:35,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:35,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:35,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:35,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:35,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:35,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:35,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:35,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:35,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:35,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:35,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:35,179 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:33:36,701 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:33:37,180 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:33:38,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:38,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:38,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:38,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:38,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:38,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:38,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:38,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:38,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:38,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:38,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:38,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:38,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:38,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:38,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:38,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:40,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:33:40,146 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:33:40,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:40,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:40,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:40,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:40,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:40,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:40,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:40,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:40,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:40,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:40,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:40,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:40,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:40,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:40,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:40,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:41,181 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:33:42,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:42,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:42,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:42,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:42,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:42,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:42,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:42,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:42,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:42,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:42,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:42,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:42,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:42,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:42,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:42,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:43,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:43,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:43,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:43,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:43,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:43,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:43,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:43,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:43,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:43,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:43,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:43,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:43,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:43,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:43,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:43,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:43,182 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:33:44,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:44,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:44,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:44,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:44,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:44,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:44,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:44,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:44,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:44,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:44,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:44,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:44,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:44,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:44,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:44,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,183 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:33:45,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:45,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:47,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:48,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:48,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:48,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:48,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:48,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:48,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:48,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:48,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:48,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:48,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:48,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:48,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:48,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:48,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:48,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:48,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,185 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:33:49,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:49,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:50,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,187 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:33:51,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:51,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:52,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,188 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:33:53,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:53,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:55,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:33:55,374 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:33:56,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:56,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:57,189 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:33:57,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:57,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:57,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:57,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:57,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:57,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:57,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:57,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:57,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:57,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:57,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:57,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:57,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:57,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:57,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:57,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:58,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:58,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:58,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:58,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:58,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:58,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:58,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:58,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:58,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:58,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:58,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:58,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:58,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:58,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:58,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:58,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:33:59,190 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:00,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:00,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:01,191 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:01,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:01,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:01,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:01,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:01,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:01,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:01,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:01,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:01,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:01,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:01,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:01,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:01,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:01,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:01,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:01,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:02,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,192 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:03,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:03,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:04,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:04,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:04,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:04,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:04,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:04,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:04,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:04,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:04,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:04,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:04,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:04,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:04,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:04,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:04,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:04,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,193 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:05,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:05,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,782 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:34:06,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:06,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:07,194 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:08,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:08,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:08,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:08,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:08,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:08,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:08,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:08,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:08,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:08,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:08,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:08,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:08,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:08,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:08,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:08,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:09,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:09,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:09,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:09,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:09,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:09,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:09,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:09,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:09,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:09,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:09,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:09,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:09,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:09,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:09,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:09,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:09,195 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:10,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:34:10,734 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:34:13,197 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:15,198 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:17,199 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:19,200 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:21,200 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:21,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:21,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:21,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:21,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:21,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:21,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:21,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:21,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:21,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:21,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:21,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:21,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:21,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:21,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:21,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:21,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:34:23,201 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:25,202 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:25,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:34:25,970 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:34:27,203 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:29,204 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:31,205 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:33,206 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:35,207 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:36,861 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:34:37,207 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:39,208 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:41,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:34:41,191 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:34:41,209 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:43,210 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:45,211 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:47,212 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:49,213 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:56,215 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:34:56,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:34:56,439 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:35:00,217 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:35:04,219 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:35:06,220 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:35:06,939 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:35:10,221 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:35:11,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:35:11,766 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:35:13,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:13,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:13,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:13,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:13,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:13,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:13,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:13,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:13,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:13,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:13,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:13,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:13,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:13,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:13,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:13,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:14,223 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:35:16,224 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:35:17,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:17,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:17,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:17,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:17,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:17,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:17,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:17,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:17,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:17,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:17,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:17,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:17,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:17,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:17,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:17,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:20,226 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:35:22,227 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:35:26,228 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:35:26,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:35:26,988 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:35:28,229 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:35:29,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:29,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:29,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:29,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:29,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:29,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:29,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:29,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:29,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:29,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:29,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:29,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:29,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:29,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:29,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:29,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:30,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:30,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:30,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:30,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:30,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:30,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:30,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:30,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:30,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:30,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:30,231 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:35:30,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:30,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:30,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:30,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:30,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:30,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:31,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:33,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,233 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:35:34,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:34,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,234 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:35:36,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:36,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:37,018 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:35:38,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:38,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:38,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:38,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:38,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:38,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:38,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:38,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:38,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:38,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:38,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:38,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:38,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:38,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:38,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:38,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:38,235 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:35:39,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:39,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:40,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:40,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:40,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:40,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:40,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:40,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:40,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:40,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:40,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:40,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:40,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:40,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:40,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:40,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:40,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:40,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:42,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:42,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:42,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:42,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:42,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:42,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:42,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:42,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:42,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:42,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:42,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:42,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:42,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:42,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:42,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:42,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:42,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:35:42,223 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:35:42,237 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:35:43,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:43,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,238 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:35:44,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:44,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:45,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:46,239 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:35:46,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:46,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:46,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:46,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:46,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:46,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:46,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:46,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:46,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:46,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:46,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:46,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:46,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:46,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:46,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:46,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:47,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:47,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:47,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:47,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:47,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:47,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:47,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:47,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:47,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:47,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:47,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:47,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:47,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:47,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:47,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:47,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:48,240 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:35:48,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:48,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:48,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:48,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:48,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:48,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:48,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:48,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:48,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:48,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:48,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:48,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:48,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:48,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:48,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:48,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:48,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:48,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:48,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:48,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,368 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 13:35:49,600 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 13:35:49,667 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 13:35:49,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:49,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:50,241 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:35:50,241 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 13:35:51,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:51,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:52,242 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:35:54,243 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:35:55,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:55,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,244 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:35:56,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:56,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:57,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:57,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:57,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:57,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:57,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:57,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:57,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:57,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:57,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:57,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:57,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:57,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:57,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:57,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:57,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:57,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:35:57,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:35:57,530 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:35:58,244 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:36:00,245 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:36:00,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:00,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:00,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:00,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:00,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:00,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:00,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:00,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:00,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:00,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:00,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:00,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:00,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:00,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:00,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:00,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:02,247 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:36:02,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:02,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:02,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:02,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:02,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:02,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:02,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:02,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:02,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:02,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:02,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:02,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:02,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:02,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:02,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:02,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:04,247 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:36:06,248 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:36:07,122 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:36:07,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:07,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:07,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:07,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:07,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:07,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:07,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:07,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:07,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:07,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:07,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:07,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:07,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:07,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:07,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:07,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:36:08,250 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:36:10,251 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:36:12,252 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:36:12,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:36:12,755 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:36:14,252 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:36:16,253 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:36:18,254 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:36:20,255 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:36:22,256 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:36:24,257 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:36:26,258 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:36:27,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:36:27,970 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:36:29,259 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:36:31,260 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:36:33,261 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:36:35,262 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:36:37,201 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:36:37,263 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:36:41,264 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:36:43,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:36:43,197 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:36:45,266 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:36:49,268 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:36:53,269 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:36:55,270 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:36:58,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:36:58,414 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:36:59,272 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:03,274 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:05,274 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:06,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:06,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:06,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:06,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:06,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:06,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:06,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:06,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:06,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:06,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:06,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:06,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:06,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:06,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:06,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:06,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:07,277 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:37:07,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:07,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:07,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:07,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:07,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:07,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:07,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:07,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:07,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:07,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:07,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:07,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:07,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:07,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:07,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:07,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:09,277 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:11,278 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:13,279 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:13,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:37:13,638 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:37:17,281 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:18,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:18,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:18,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:18,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:18,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:18,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:18,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:18,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:18,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:18,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:18,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:18,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:18,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:18,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:18,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:18,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:19,282 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:20,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:20,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:20,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:20,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:20,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:20,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:20,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:20,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:20,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:20,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:20,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:20,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:20,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:20,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:20,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:20,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:21,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:21,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:21,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:21,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:21,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:21,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:21,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:21,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:21,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:21,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:21,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:21,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:21,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:21,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:21,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:21,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:21,282 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:22,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:22,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:22,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:22,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:22,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:22,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:22,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:22,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:22,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:22,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:22,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:22,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:22,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:22,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:22,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:22,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:23,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:23,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:23,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:23,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:23,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:23,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:23,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:23,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:23,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:23,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:23,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:23,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:23,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:23,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:23,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:23,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:24,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:25,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:25,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:25,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:25,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:25,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:25,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:25,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:25,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:25,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:25,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:25,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:25,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:25,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:25,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:25,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:25,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:25,284 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:26,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:26,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:26,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:26,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:26,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:26,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:26,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:26,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:26,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:26,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:26,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:26,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:26,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:26,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:26,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:26,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:27,285 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:27,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:27,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:27,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:27,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:27,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:27,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:27,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:27,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:27,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:27,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:27,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:27,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:27,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:27,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:27,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:27,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,448 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:28,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:37:28,953 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:37:29,285 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:31,286 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:32,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:32,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,287 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:33,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:33,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:34,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,288 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:35,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:35,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:36,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:36,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:36,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:36,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:36,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:36,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:36,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:36,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:36,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:36,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:36,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:36,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:36,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:36,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:36,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:36,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,356 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:37:37,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:37,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:38,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:39,289 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:40,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:40,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:40,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:40,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:40,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:40,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:40,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:40,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:40,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:40,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:40,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:40,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:40,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:40,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:40,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:40,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:41,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:41,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:41,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:41,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:41,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:41,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:41,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:41,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:41,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:41,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:41,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:41,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:41,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:41,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:41,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:41,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:41,290 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:42,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:42,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:43,291 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:43,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:43,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:43,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:43,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:43,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:43,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:43,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:43,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:43,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:43,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:43,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:43,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:43,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:43,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:43,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:43,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:44,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:37:44,270 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:37:45,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:45,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:45,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:45,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:45,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:45,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:45,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:45,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:45,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:45,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:45,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:45,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:45,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:45,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:45,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:45,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:45,291 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:46,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:46,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,292 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:47,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:47,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:48,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:48,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:48,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:48,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:48,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:48,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:48,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:48,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:48,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:48,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:48,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:48,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:48,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:48,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:48,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:48,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:49,293 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:50,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:50,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:50,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:50,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:50,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:50,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:50,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:50,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:50,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:50,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:50,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:50,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:50,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:50,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:50,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:50,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:51,294 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:52,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:52,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:52,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:52,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:52,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:52,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:52,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:52,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:52,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:52,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:52,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:52,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:52,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:52,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:52,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:52,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:53,294 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:55,295 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:57,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:57,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:57,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:57,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:57,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:57,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:57,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:57,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:57,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:57,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:57,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:57,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:57,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:57,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:57,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:57,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:37:57,296 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:59,297 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:37:59,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:37:59,502 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:38:01,297 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:38:03,298 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:38:05,299 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:38:07,300 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:38:07,436 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:38:09,301 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:38:11,302 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:38:13,302 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:38:14,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:38:14,755 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:38:15,303 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:38:17,304 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:38:19,305 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:38:21,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:38:25,307 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:38:29,309 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:38:30,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:38:30,022 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:38:33,311 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:38:37,312 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:38:37,514 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:38:41,314 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:38:43,315 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:38:45,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:38:45,249 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:38:47,317 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:38:50,318 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:38:54,320 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:38:55,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:55,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:55,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:55,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:55,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:55,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:55,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:55,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:55,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:55,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:55,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:55,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:55,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:55,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:55,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:55,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:56,321 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:38:59,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:59,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:59,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:59,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:59,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:59,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:59,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:59,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:59,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:59,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:59,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:59,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:59,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:59,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:59,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:38:59,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:00,322 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:00,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:39:00,469 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:39:02,323 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:04,324 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:06,325 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:07,594 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:39:08,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:08,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:08,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:08,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:08,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:08,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:08,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:08,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:08,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:08,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:08,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:08,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:08,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:08,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:08,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:08,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:09,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:09,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:09,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:09,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:09,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:09,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:09,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:09,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:09,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:09,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:09,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:09,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:09,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:09,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:09,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:09,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,327 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:10,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:10,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:11,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:11,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:11,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:11,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:11,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:11,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:11,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:11,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:11,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:11,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:11,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:11,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:11,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:11,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:11,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:11,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:12,328 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:12,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:12,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:12,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:12,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:12,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:12,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:12,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:12,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:12,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:12,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:12,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:12,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:12,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:12,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:12,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:12,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:13,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:14,328 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:15,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:39:15,783 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:39:16,329 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:16,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:16,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:16,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:16,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:16,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:16,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:16,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:16,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:16,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:16,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:16,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:16,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:16,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:16,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:16,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:16,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:17,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:17,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:17,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:17,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:17,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:17,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:17,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:17,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:17,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:17,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:17,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:17,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:17,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:17,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:17,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:17,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,330 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:18,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:18,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,173 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 13:39:19,393 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 13:39:19,471 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 13:39:19,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:19,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:20,415 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:20,415 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 13:39:20,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:20,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:20,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:20,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:20,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:20,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:20,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:20,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:20,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:20,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:20,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:20,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:20,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:20,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:20,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:20,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:21,415 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:21,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:21,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:21,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:21,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:21,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:21,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:21,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:21,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:21,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:21,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:21,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:21,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:21,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:21,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:21,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:21,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:22,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:23,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:23,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:23,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:23,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:23,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:23,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:23,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:23,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:23,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:23,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:23,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:23,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:23,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:23,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:23,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:23,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,416 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:24,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:24,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:25,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:26,417 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:26,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:26,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:26,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:26,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:26,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:26,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:26,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:26,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:26,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:26,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:26,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:26,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:26,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:26,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:26,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:26,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:27,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:28,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:28,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:28,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:28,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:28,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:28,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:28,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:28,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:28,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:28,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:28,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:28,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:28,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:28,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:28,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:28,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:28,419 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:29,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:29,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:29,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:29,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:29,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:29,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:29,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:29,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:29,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:29,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:29,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:29,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:29,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:29,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:29,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:29,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:30,419 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:30,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:30,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:30,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:30,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:30,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:30,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:30,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:30,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:30,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:30,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:30,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:30,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:30,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:30,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:30,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:30,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:39:31,039 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:39:31,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:31,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:32,421 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:32,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:32,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:32,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:32,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:32,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:32,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:32,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:32,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:32,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:32,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:32,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:32,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:32,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:32,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:32,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:32,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,421 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:34,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:34,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:35,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:35,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:35,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:35,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:35,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:35,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:35,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:35,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:35,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:35,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:35,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:35,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:35,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:35,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:35,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:35,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:36,422 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:36,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:36,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:36,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:36,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:36,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:36,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:36,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:36,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:36,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:36,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:36,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:36,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:36,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:36,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:36,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:36,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:37,720 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:39:37,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:37,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:37,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:37,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:37,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:37,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:37,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:37,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:37,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:37,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:37,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:37,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:37,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:37,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:37,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:37,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,423 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:38,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:38,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:39,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:39,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:39,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:39,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:39,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:39,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:39,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:39,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:39,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:39,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:39,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:39,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:39,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:39,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:39,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:39,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:40,424 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:42,425 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:44,426 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:44,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:44,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:44,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:44,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:44,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:44,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:44,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:44,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:44,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:44,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:44,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:44,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:44,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:44,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:44,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:44,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:39:46,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:39:46,265 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:39:46,427 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:48,428 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:50,429 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:52,430 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:54,430 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:56,431 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:39:58,432 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:40:00,433 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:40:01,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:40:01,514 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:40:02,434 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:40:04,435 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:40:06,435 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:40:07,799 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:40:12,438 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:40:16,440 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:40:16,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:40:16,739 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:40:18,441 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:40:22,442 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:40:26,444 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:40:28,445 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:40:31,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:40:31,964 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:40:32,447 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:40:34,447 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:40:37,878 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:40:38,449 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:40:40,450 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:40:44,451 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:40:44,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:44,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:44,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:44,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:44,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:44,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:44,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:44,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:44,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:44,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:44,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:44,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:44,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:44,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:44,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:44,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:46,452 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:40:47,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:40:47,229 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:40:48,455 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:40:51,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:51,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:51,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:51,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:51,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:51,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:51,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:51,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:51,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:51,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:51,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:51,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:51,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:51,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:51,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:51,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:52,457 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:40:54,458 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:40:56,459 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:40:56,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:56,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:56,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:56,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:56,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:56,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:56,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:56,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:56,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:56,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:56,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:56,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:56,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:56,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:56,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:56,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:57,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:57,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:57,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:57,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:57,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:57,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:57,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:57,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:57,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:57,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:57,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:57,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:57,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:57,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:57,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:57,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:58,460 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:40:59,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:40:59,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:00,461 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:41:01,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:01,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:02,461 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:41:02,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:41:02,566 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:41:03,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:03,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:04,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:04,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:04,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:04,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:04,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:04,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:04,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:04,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:04,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:04,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:04,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:04,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:04,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:04,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:04,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:04,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:04,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:04,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:04,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:04,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:04,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:04,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:04,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:04,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:05,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:06,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:06,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:06,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:06,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:06,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:06,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:06,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:06,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:06,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:06,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:06,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:06,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:06,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:06,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:06,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:06,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:07,463 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:41:07,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:07,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:07,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:07,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:07,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:07,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:07,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:07,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:07,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:07,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:07,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:07,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:07,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:07,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:07,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:07,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:07,958 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:41:08,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:08,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:08,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:08,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:08,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:08,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:08,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:08,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:08,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:08,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:08,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:08,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:08,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:08,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:08,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:08,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:09,464 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:41:10,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:10,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:10,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:10,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:10,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:10,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:10,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:10,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:10,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:10,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:10,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:10,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:10,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:10,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:10,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:10,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:11,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:11,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:11,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:11,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:11,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:11,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:11,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:11,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:11,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:11,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:11,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:11,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:11,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:11,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:11,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:11,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:11,465 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:41:12,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:12,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:13,466 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:41:13,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:13,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:14,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:15,467 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:41:17,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,468 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:41:17,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:41:17,792 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:41:17,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:17,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:19,469 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:41:19,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:19,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:19,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:19,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:19,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:19,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:19,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:19,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:19,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:19,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:19,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:19,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:19,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:19,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:19,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:19,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:20,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,470 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:41:21,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:21,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:22,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:22,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:22,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:22,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:22,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:22,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:22,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:22,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:22,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:22,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:22,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:22,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:22,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:22,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:22,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:22,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:23,471 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:41:25,472 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:41:25,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:25,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:25,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:25,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:25,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:25,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:25,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:25,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:25,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:25,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:25,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:25,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:25,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:25,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:25,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:25,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:26,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:26,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:26,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:26,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:26,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:26,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:26,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:26,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:26,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:26,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:26,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:26,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:26,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:26,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:26,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:26,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:27,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:27,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:27,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:27,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:27,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:27,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:27,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:27,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:27,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:27,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:27,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:27,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:27,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:27,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:27,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:27,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:27,472 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:41:29,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:41:29,473 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:41:31,474 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:41:33,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:41:33,050 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:41:33,475 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:41:35,476 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:41:37,477 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:41:38,036 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:41:39,478 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:41:41,479 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:41:43,480 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:41:45,481 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:41:47,482 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:41:48,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:41:48,273 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:41:49,483 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:41:55,485 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:41:59,487 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:42:01,488 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:42:03,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:42:03,499 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:42:05,490 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:42:08,118 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:42:09,491 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:42:11,492 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:42:15,494 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:42:18,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:42:18,846 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:42:19,496 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:42:21,497 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:42:25,499 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:42:27,500 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:42:29,501 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:42:29,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:29,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:29,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:29,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:29,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:29,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:29,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:29,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:29,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:29,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:29,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:29,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:29,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:29,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:29,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:29,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:33,503 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:42:34,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:42:34,158 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:42:35,503 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:42:37,504 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:42:38,197 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:42:41,506 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:42:41,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:41,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:41,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:41,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:41,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:41,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:41,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:41,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:41,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:41,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:41,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:41,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:41,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:41,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:41,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:41,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:43,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:43,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:43,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:43,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:43,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:43,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:43,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:43,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:43,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:43,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:43,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:43,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:43,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:43,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:43,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:43,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:43,507 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:42:45,508 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:42:46,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:46,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:46,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:46,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:46,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:46,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:46,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:46,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:46,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:46,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:46,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:46,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:46,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:46,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:46,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:46,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:47,509 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:42:47,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:47,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:47,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:47,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:47,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:47,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:47,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:47,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:47,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:47,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:47,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:47,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:47,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:47,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:47,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:47,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:47,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:47,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:47,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:47,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:47,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:48,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:48,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:48,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:48,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:48,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:48,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:48,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:48,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:48,310 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 13:42:48,462 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 13:42:48,516 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 13:42:48,517 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 13:42:49,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:42:49,452 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:42:49,517 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:42:50,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,517 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:42:50,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:50,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,518 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:42:51,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:51,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:52,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:53,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:53,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:53,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:53,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:53,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:53,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:53,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:53,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:53,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:53,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:53,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:53,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:53,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:53,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:53,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:53,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:53,519 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:42:55,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,448 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,520 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:42:55,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:55,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:56,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:57,522 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:42:58,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:58,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:58,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:58,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:58,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:58,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:58,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:58,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:58,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:58,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:58,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:58,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:58,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:58,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:58,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:58,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:59,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:59,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:59,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:59,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:59,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:59,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:59,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:59,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:59,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:59,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:59,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:59,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:59,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:59,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:59,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:59,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:42:59,523 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:43:00,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:00,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:00,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:00,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:00,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:00,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:00,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:00,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:00,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:00,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:00,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:00,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:00,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:00,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:00,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:00,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:01,524 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:43:03,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:03,525 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:43:04,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:04,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:43:04,727 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:43:05,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:05,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:05,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:05,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:05,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:05,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:05,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:05,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:05,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:05,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:05,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:05,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:05,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:05,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:05,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:05,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:05,526 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:43:06,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:06,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:07,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:07,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:07,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:07,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:07,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:07,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:07,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:07,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:07,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:07,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:07,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:07,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:07,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:07,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:07,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:07,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:07,526 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:43:08,572 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:43:09,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:09,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:09,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:09,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:09,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:09,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:09,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:09,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:09,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:09,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:09,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:09,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:09,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:09,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:09,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:09,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:09,527 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:43:10,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:10,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:11,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:11,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:11,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:11,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:11,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:11,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:11,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:11,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:11,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:11,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:11,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:11,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:11,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:11,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:11,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:11,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:11,528 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:43:12,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:12,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:12,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:12,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:12,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:12,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:12,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:12,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:12,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:12,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:12,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:12,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:12,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:12,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:12,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:12,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:13,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:13,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:13,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:13,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:13,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:13,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:13,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:13,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:13,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:13,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:13,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:13,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:13,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:13,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:13,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:13,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:13,529 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:43:14,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:14,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:15,530 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:43:16,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:16,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:16,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:16,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:16,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:16,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:16,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:16,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:16,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:16,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:16,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:16,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:16,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:16,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:16,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:16,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:17,531 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:43:17,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:17,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:17,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:17,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:17,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:17,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:17,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:17,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:17,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:17,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:17,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:17,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:17,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:17,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:17,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:17,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:18,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:18,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:18,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:18,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:18,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:18,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:18,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:18,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:18,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:18,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:18,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:18,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:18,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:18,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:18,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:18,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:43:19,531 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:43:19,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:43:19,952 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:43:21,532 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:43:23,533 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:43:25,534 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:43:28,535 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:43:30,536 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:43:32,537 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:43:34,538 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:43:35,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:43:35,205 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:43:38,652 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:43:40,540 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:43:44,542 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:43:48,544 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:43:50,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:43:50,433 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:43:52,546 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:43:54,546 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:43:58,548 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:02,550 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:04,551 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:05,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:44:05,764 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:44:06,551 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:08,731 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:44:10,553 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:12,554 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:16,556 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:18,557 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:20,558 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:21,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:44:21,003 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:44:21,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:21,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:21,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:21,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:21,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:21,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:21,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:21,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:21,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:21,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:21,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:21,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:21,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:21,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:21,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:21,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:22,559 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:26,561 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:28,561 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:30,562 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:32,563 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:32,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:32,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:32,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:32,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:33,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:33,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:33,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:33,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:33,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:33,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:33,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:33,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:34,564 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:35,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:35,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:35,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:35,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:35,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:35,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:35,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:35,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:35,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:35,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:35,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:35,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:35,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:35,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:35,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:35,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:36,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:44:36,232 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:44:36,565 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:37,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:37,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:38,566 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:38,810 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:44:39,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:39,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,567 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:40,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:40,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:41,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:42,567 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:42,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:42,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:42,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:42,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:42,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:42,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:42,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:42,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:42,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:42,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:42,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:42,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:42,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:42,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:42,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:42,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:43,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:43,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:43,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:43,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:43,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:43,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:43,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:43,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:43,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:43,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:43,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:43,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:43,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:43,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:43,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:43,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:44,569 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:44,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:44,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:44,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:44,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:44,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:44,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:44,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:44,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:44,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:44,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:44,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:44,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:44,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:44,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:44,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:44,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:45,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,570 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:46,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:46,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:47,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:47,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:47,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:47,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:47,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:47,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:47,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:47,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:47,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:47,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:47,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:47,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:47,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:47,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:47,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:47,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:48,571 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:48,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:48,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:48,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:48,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:48,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:48,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:48,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:48,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:48,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:48,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:48,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:48,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:48,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:48,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:48,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:48,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:49,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:49,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:49,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:49,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:49,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:49,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:49,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:49,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:49,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:49,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:49,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:49,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:49,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:49,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:49,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:49,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:50,572 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:51,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:51,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:51,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:51,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:51,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:51,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:51,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:51,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:51,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:51,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:51,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:51,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:51,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:51,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:51,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:51,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:51,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:44:51,474 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:44:52,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,573 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:52,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:52,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:53,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:53,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:53,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:53,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:53,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:53,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:53,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:53,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:53,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:53,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:53,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:53,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:53,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:53,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:53,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:53,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,574 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:54,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:54,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:55,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:56,575 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:44:57,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:57,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:57,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:57,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:57,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:57,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:57,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:57,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:57,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:57,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:57,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:57,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:57,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:57,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:57,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:57,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:57,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:57,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:57,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:57,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:57,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:57,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:44:58,575 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:45:00,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,576 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:45:00,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:00,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:01,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:02,577 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:45:02,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:02,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:02,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:02,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:02,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:02,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:02,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:02,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:02,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:02,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:02,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:02,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:02,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:02,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:02,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:02,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:03,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:03,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:03,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:03,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:03,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:03,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:03,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:03,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:03,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:03,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:03,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:03,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:03,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:03,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:03,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:03,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:04,578 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:45:04,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:04,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:04,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:04,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:04,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:04,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:04,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:04,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:04,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:04,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:04,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:04,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:04,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:04,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:04,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:04,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:05,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:05,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:05,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:05,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:05,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:05,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:05,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:05,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:05,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:05,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:05,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:05,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:05,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:05,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:05,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:05,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:06,579 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:45:06,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:45:06,727 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:45:08,580 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:45:08,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:08,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:08,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:08,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:08,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:08,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:08,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:08,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:08,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:08,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:08,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:08,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:08,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:08,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:08,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:08,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:45:08,896 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:45:10,581 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:45:12,582 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:45:14,582 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:45:16,583 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:45:18,584 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:45:20,585 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:45:21,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:45:21,948 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:45:24,587 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:45:28,588 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:45:32,590 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:45:36,591 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:45:37,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:45:37,206 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:45:38,975 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:45:40,593 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:45:42,594 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:45:46,595 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:45:48,596 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:45:52,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:45:52,421 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:45:52,598 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:45:54,599 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:45:56,600 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:00,601 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:02,602 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:05,603 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:07,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:07,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:07,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:07,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:07,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:07,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:07,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:07,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:07,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:07,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:07,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:07,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:07,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:07,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:07,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:07,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:07,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:46:07,647 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:46:09,059 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:46:09,605 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:11,606 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:13,607 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:15,607 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:17,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:17,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:17,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:17,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:17,608 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:18,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:18,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:18,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:18,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:18,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:18,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:18,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:18,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:18,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 13:46:18,354 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 13:46:18,562 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 13:46:18,634 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 13:46:19,633 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:19,633 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 13:46:20,634 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:21,634 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:23,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:46:23,065 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:46:23,635 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:25,635 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:27,636 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:29,637 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:31,638 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:33,638 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:35,639 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:37,640 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:38,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:46:38,291 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:46:39,189 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:46:39,641 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:41,642 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:43,642 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:45,643 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:47,644 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:49,645 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:51,645 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:53,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:46:53,555 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:46:53,646 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:55,647 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:57,647 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:46:59,648 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:01,649 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:03,650 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:05,651 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:07,652 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:08,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:47:08,875 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:47:09,268 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:47:09,652 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:11,653 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:13,654 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:15,655 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:17,656 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:19,657 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:21,657 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:23,658 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:24,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:47:24,233 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:47:25,659 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:27,660 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:29,661 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:31,661 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:33,662 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:35,663 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:37,664 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:39,351 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:47:39,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:47:39,474 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:47:39,665 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:42,666 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:44,667 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:46,667 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:48,668 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:50,669 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:52,670 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:54,671 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:54,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:47:54,699 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:47:56,672 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:47:58,673 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:00,673 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:02,674 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:04,675 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:06,676 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:08,677 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:09,432 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:48:09,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:48:09,990 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:48:10,678 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:12,679 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:14,680 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:16,681 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:18,681 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:20,682 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:22,683 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:24,684 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:25,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:48:25,326 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:48:26,685 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:28,686 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:30,686 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:32,687 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:34,688 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:36,689 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:38,690 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:39,512 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:48:40,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:48:40,592 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:48:40,690 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:42,691 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:44,692 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:46,693 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:48,694 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:50,695 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:52,695 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:54,696 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:55,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:48:55,892 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:48:56,697 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:48:58,698 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:00,699 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:02,700 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:04,700 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:06,701 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:08,702 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:09,593 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:49:11,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:49:11,121 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:49:11,703 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:13,704 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:15,705 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:17,706 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:19,706 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:21,707 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:23,708 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:25,709 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:26,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:49:26,414 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:49:27,709 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:29,710 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:31,711 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:33,712 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:35,713 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:37,713 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:39,674 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:49:39,714 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:41,715 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:41,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:49:41,778 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:49:43,716 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:45,717 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:47,717 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:49,718 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:51,719 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:53,720 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:55,720 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:57,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:49:57,154 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:49:57,721 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:49:59,722 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:01,723 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:03,723 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:05,724 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:07,725 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:09,726 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:09,754 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:50:11,726 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:12,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:50:12,383 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:50:13,727 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:15,728 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:17,729 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:19,730 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:21,731 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:23,731 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:25,732 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:27,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:50:27,605 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:50:27,733 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:29,734 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:31,734 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:33,735 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:35,736 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:38,738 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:39,832 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:50:40,739 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:42,740 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:42,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:50:42,893 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:50:44,741 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:46,742 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:48,743 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:50,744 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:52,745 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:54,746 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:56,747 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:50:58,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:50:58,115 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:50:58,748 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:00,749 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:02,750 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:04,751 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:06,752 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:08,753 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:09,912 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:51:10,754 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:12,755 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:13,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:51:13,370 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:51:14,756 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:16,757 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:18,758 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:20,759 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:22,760 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:24,761 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:26,761 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:28,762 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:28,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:51:28,773 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:51:30,763 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:32,764 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:34,765 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:36,766 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:38,767 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:39,992 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:51:40,767 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:42,768 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:44,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:51:44,097 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:51:44,769 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:46,770 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:48,770 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:50,771 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:52,772 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:54,773 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:56,774 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:58,774 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:51:59,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:51:59,417 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:52:00,775 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:02,776 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:04,777 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:06,778 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:08,778 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:10,072 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:52:10,779 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:12,780 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:14,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:52:14,702 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:52:14,781 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:16,782 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:19,783 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:21,784 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:23,785 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:25,785 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:27,786 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:29,787 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:30,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:52:30,002 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:52:31,788 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:33,789 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:35,790 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:37,791 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:39,791 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:40,151 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:52:41,792 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:43,793 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:45,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:52:45,233 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:52:45,794 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:47,795 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:49,796 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:51,796 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:53,797 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:55,798 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:57,799 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:52:59,800 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:00,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:53:00,585 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:53:01,801 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:03,802 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:05,802 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:07,803 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:09,804 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:10,232 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:53:11,805 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:13,806 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:15,807 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:15,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:53:15,880 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:53:17,808 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:19,809 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:21,809 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:23,810 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:25,811 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:27,812 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:29,813 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:31,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:53:31,136 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:53:31,814 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:33,815 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:35,815 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:37,816 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:39,817 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:40,312 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:53:41,818 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:43,819 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:45,820 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:46,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:53:46,468 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:53:47,820 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:49,821 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:51,822 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:53,823 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:55,824 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:57,825 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:53:59,826 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:01,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:54:01,693 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:54:02,827 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:04,828 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:06,829 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:08,830 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:10,392 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:54:10,831 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:12,831 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:14,832 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:16,833 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:16,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:54:16,969 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:54:18,834 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:20,835 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:22,836 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:24,837 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:26,838 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:28,839 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:30,839 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:32,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:54:32,197 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:54:32,841 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:34,842 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:36,843 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:38,843 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:40,473 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:54:40,844 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:42,845 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:44,846 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:46,847 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:47,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:54:47,566 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:54:48,848 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:50,848 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:52,849 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:54,850 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:56,851 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:54:58,852 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:00,853 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:02,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:55:02,849 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:55:02,862 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:04,863 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:06,863 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:08,864 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:10,553 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:55:10,865 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:12,866 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:14,867 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:16,868 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:18,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:55:18,105 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:55:18,869 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:20,870 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:22,871 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:25,872 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:27,873 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:29,874 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:31,875 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:33,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:55:33,323 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:55:33,876 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:35,877 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:37,878 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:39,879 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:40,633 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:55:41,880 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:43,881 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:45,882 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:47,883 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:48,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:55:48,641 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:55:49,884 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:51,885 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:53,886 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:55,887 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:57,888 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:55:59,889 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:01,890 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:03,891 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:03,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:56:03,946 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:56:05,892 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:07,893 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:09,894 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:10,713 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:56:11,895 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:13,896 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:15,897 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:17,898 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:19,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:56:19,234 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:56:19,898 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:21,900 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:23,900 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:25,901 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:27,902 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:29,903 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:31,904 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:33,905 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:34,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:56:34,463 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:56:35,906 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:37,907 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:39,908 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:40,792 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:56:41,909 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:43,910 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:45,911 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:47,911 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:49,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:56:49,802 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:56:49,912 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:51,913 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:53,914 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:55,915 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:57,916 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:56:59,917 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:01,918 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:03,919 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:05,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:57:05,078 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:57:05,920 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:07,921 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:10,873 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:57:10,922 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:12,923 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:14,924 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:16,925 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:18,926 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:20,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:57:20,324 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:57:20,927 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:22,928 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:24,929 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:26,930 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:28,931 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:30,932 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:32,933 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:34,934 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:35,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:57:35,626 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:57:36,934 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:38,935 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:40,936 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:40,954 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:57:42,937 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:44,938 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:46,939 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:48,940 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:50,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:57:50,938 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:57:50,942 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:52,943 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:54,944 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:56,945 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:57:58,946 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:00,946 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:02,947 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:04,948 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:06,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:58:06,167 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:58:06,949 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:08,950 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:10,951 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:11,034 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:58:12,952 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:14,953 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:16,954 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:18,954 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:20,955 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:21,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:58:21,395 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:58:22,956 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:24,957 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:26,958 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:28,959 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:30,960 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:32,960 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:34,961 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:36,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:58:36,726 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:58:36,962 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:38,963 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:40,964 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:41,114 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:58:43,965 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:45,966 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:47,967 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:49,967 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:51,968 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:52,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:58:52,053 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:58:53,969 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:55,970 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:57,971 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:58:59,972 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:01,973 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:03,974 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:05,975 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:07,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:59:07,276 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:59:07,975 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:09,976 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:11,193 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:59:11,977 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:13,978 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:15,979 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:17,980 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:19,981 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:21,981 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:22,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:59:22,603 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:59:23,982 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:25,983 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:27,984 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:29,985 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:31,986 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:33,986 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:35,987 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:37,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:59:37,885 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:59:37,988 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:39,989 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:41,274 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 13:59:41,989 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:43,990 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:45,991 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:47,992 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:49,993 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:51,994 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:53,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 13:59:53,117 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 13:59:53,994 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:55,995 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:57,996 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 13:59:59,997 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:01,998 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:03,999 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:05,999 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:08,000 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:08,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:00:08,369 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:00:10,001 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:11,354 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:00:12,002 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:15,003 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:17,004 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:19,005 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:21,006 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:23,007 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:23,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:00:23,678 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:00:25,007 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:27,008 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:29,009 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:31,010 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:33,011 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:35,012 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:37,012 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:39,013 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:39,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:00:39,034 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:00:41,014 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:41,434 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:00:43,015 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:45,016 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:47,017 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:49,017 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:51,018 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:53,019 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:54,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:00:54,263 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:00:55,020 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:57,021 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:00:59,022 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:01,023 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:03,024 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:05,024 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:07,025 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:09,026 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:09,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:01:09,685 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:01:11,027 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:11,515 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:01:13,028 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:15,029 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:17,030 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:19,030 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:21,031 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:23,032 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:24,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:01:24,908 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:01:25,033 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:27,034 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:29,035 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:31,036 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:33,036 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:35,037 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:37,038 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:39,039 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:40,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:01:40,134 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:01:41,040 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:41,594 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:01:43,041 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:45,042 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:47,043 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:49,043 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:51,044 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:53,045 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:55,046 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:55,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:01:55,423 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:01:57,047 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:01:59,048 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:01,049 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:03,049 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:05,050 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:07,051 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:09,052 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:10,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:02:10,697 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:02:11,053 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:11,673 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:02:13,053 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:16,055 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:18,056 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:20,057 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:22,057 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:24,058 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:25,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:02:25,935 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:02:26,059 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:28,060 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:30,061 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:32,062 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:34,063 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:36,064 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:38,065 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:40,066 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:41,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:02:41,150 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:02:41,753 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:02:42,067 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:44,068 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:46,069 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:48,070 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:50,071 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:52,072 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:54,073 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:56,074 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:02:56,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:02:56,490 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:02:58,075 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:00,076 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:02,076 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:04,077 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:06,078 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:08,079 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:10,080 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:11,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:03:11,716 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:03:11,940 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:03:12,081 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:14,082 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:17,083 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:19,084 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:21,085 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:23,086 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:25,086 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:27,087 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:27,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:03:27,095 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:03:29,088 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:31,089 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:33,090 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:35,091 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:37,092 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:39,093 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:41,094 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:41,910 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:03:42,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:03:42,395 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:03:43,094 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:45,095 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:47,096 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:49,097 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:51,098 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:53,099 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:55,100 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:57,101 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:03:57,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:03:57,624 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:03:59,101 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:01,102 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:03,103 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:05,104 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:07,105 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:09,105 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:11,106 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:11,988 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:04:12,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:04:12,984 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:04:13,107 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:15,108 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:17,109 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:19,110 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:21,110 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:23,111 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:25,112 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:28,113 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:28,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:04:28,214 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:04:30,114 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:32,115 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:34,116 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:36,117 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:38,117 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:40,118 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:42,066 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:04:42,119 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:43,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:04:43,536 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:04:44,120 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:46,121 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:48,122 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:50,122 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:52,124 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:54,124 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:56,125 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:58,126 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:04:58,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:04:58,759 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:05:00,127 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:02,128 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:04,129 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:06,129 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:08,131 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:10,131 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:12,132 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:12,145 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:05:14,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:05:14,062 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:05:14,133 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:16,134 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:18,135 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:20,136 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:22,137 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:24,138 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:26,138 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:28,139 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:29,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:05:29,285 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:05:30,140 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:32,141 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:34,142 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:36,143 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:38,144 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:40,145 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:42,145 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:42,223 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:05:44,146 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:44,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:05:44,569 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:05:46,147 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:48,148 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:50,149 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:52,150 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:54,151 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:56,152 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:58,153 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:05:59,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:05:59,792 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:06:00,154 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:02,155 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:05,156 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:07,157 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:09,158 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:11,159 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:12,303 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:06:13,160 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:15,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:06:15,080 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:06:15,160 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:17,161 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:19,162 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:21,163 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:23,164 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:25,165 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:27,166 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:29,167 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:30,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:06:30,411 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:06:31,168 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:33,169 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:35,170 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:37,171 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:39,172 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:41,172 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:42,382 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:06:43,173 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:45,174 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:45,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:06:45,675 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:06:47,175 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:49,176 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:51,177 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:53,178 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:55,179 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:57,180 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:06:59,180 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:00,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:07:00,934 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:07:01,181 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:03,182 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:05,183 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:07,184 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:09,185 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:11,186 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:12,462 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:07:13,187 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:15,187 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:16,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:07:16,228 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:07:17,188 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:19,189 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:21,190 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:23,191 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:26,193 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:28,194 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:30,194 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:31,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:07:31,453 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:07:32,196 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:34,197 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:36,198 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:38,198 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:40,199 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:42,200 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:42,542 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:07:44,201 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:46,202 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:46,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:07:46,782 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:07:48,203 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:50,203 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:52,204 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:54,205 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:56,206 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:07:58,206 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:00,207 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:02,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:08:02,006 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:08:02,208 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:04,209 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:06,210 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:08,211 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:10,211 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:12,212 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:12,622 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:08:14,213 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:16,214 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:17,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:08:17,292 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:08:18,215 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:20,216 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:22,217 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:24,218 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:26,219 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:28,219 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:30,220 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:32,221 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:32,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:08:32,568 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:08:34,222 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:36,223 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:39,224 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:41,225 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:42,702 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:08:43,226 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:45,227 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:47,228 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:47,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:08:47,879 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:08:49,229 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:51,230 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:53,231 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:55,231 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:57,232 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:08:59,233 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:01,234 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:03,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:09:03,104 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:09:03,235 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:05,236 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:07,237 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:09,238 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:11,239 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:12,782 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:09:13,239 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:15,240 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:17,241 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:18,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:09:18,320 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:09:19,242 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:21,243 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:23,244 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:25,245 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:27,246 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:29,246 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:31,247 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:33,248 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:33,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:09:33,622 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:09:35,249 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:37,250 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:39,251 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:41,251 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:42,861 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:09:43,252 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:45,253 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:47,254 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:48,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:09:48,927 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:09:49,255 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:51,256 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:53,257 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:55,258 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:57,258 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:09:59,259 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:01,260 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:04,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:10:04,148 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:10:04,262 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:06,263 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:08,263 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:10,264 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:12,265 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:12,939 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:10:14,266 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:16,266 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:18,267 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:19,448 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:10:19,448 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:10:20,268 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:22,268 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:24,269 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:26,270 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:28,271 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:30,272 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:32,272 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:34,273 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:34,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:10:34,672 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:10:36,274 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:38,275 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:40,276 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:42,276 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:43,018 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:10:44,277 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:46,278 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:48,278 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:49,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:10:49,979 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:10:50,279 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:52,280 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:54,280 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:56,281 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:10:58,282 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:00,283 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:02,284 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:04,285 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:05,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:11:05,204 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:11:07,286 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:09,286 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:11,287 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:13,098 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:11:13,288 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:15,288 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:17,289 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:19,290 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:20,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:11:20,515 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:11:21,291 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:23,291 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:25,292 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:27,293 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:29,294 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:31,294 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:33,295 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:35,296 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:35,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:11:35,811 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:11:37,297 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:39,298 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:41,298 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:43,177 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:11:43,299 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:45,300 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:47,300 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:49,301 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:51,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:11:51,069 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:11:51,302 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:53,303 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:55,304 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:57,304 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:11:59,305 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:01,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:03,307 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:05,308 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:06,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:12:06,291 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:12:07,308 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:10,309 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:12,310 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:13,256 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:12:14,311 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:16,312 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:18,313 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:20,313 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:21,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:12:21,590 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:12:22,314 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:24,315 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:26,316 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:28,317 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:30,317 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:32,318 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:34,319 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:36,320 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:36,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:12:36,842 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:12:38,321 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:40,322 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:42,323 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:43,335 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:12:44,323 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:46,324 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:48,325 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:50,326 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:52,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:12:52,238 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:12:52,326 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:54,327 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:56,328 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:12:58,329 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:00,329 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:02,330 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:04,331 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:06,332 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:07,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:13:07,461 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:13:08,332 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:10,333 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:12,334 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:13,415 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:13:14,334 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:16,335 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:18,336 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:20,336 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:22,337 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:22,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:13:22,941 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:13:24,338 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:26,339 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:28,340 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:30,341 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:32,342 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:35,343 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:37,344 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:38,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:13:38,218 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:13:39,345 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:41,346 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:43,347 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:43,494 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:13:45,348 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:47,348 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:49,349 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:51,350 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:53,351 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:53,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:13:53,469 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:13:55,352 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:57,353 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:13:59,354 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:01,355 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:03,357 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:05,358 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:07,358 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:08,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:14:08,820 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:14:09,359 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:11,360 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:13,361 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:13,573 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:14:15,362 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:17,362 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:19,363 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:21,364 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:23,364 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:24,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:14:24,035 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:14:25,365 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:27,366 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:29,366 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:31,367 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:33,368 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:35,369 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:37,369 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:39,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:14:39,308 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:14:39,370 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:41,371 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:43,371 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:43,653 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:14:45,372 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:47,373 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:49,373 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:51,374 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:53,375 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:54,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:14:54,559 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:14:55,376 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:57,376 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:14:59,377 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:01,378 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:03,378 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:05,379 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:07,380 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:09,380 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:09,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:15:09,846 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:15:11,381 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:13,382 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:13,733 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:15:15,383 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:17,383 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:20,384 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:22,385 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:24,386 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:25,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:15:25,109 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:15:26,387 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:28,387 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:30,388 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:32,389 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:34,390 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:36,390 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:38,391 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:40,392 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:40,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:15:40,406 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:15:42,393 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:43,812 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:15:44,393 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:46,394 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:48,395 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:50,396 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:52,397 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:54,398 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:55,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:15:55,635 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:15:56,399 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:15:58,400 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:00,401 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:02,402 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:04,403 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:06,404 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:08,404 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:10,405 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:10,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:16:10,905 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:16:12,406 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:13,893 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:16:14,407 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:16,408 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:18,409 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:20,409 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:22,410 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:24,411 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:26,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:16:26,211 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:16:26,412 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:28,412 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:30,413 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:32,414 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:35,415 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:37,416 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:39,416 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:41,417 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:41,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:16:41,506 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:16:43,418 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:43,971 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:16:45,419 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:47,419 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:49,420 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:51,421 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:53,422 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:55,423 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:56,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:16:56,869 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:16:57,424 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:16:59,424 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:01,425 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:03,426 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:05,427 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:07,428 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:09,429 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:11,430 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:12,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:17:12,173 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:17:13,431 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:14,050 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:17:15,432 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:17,433 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:19,434 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:21,435 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:23,436 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:25,436 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:27,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:17:27,398 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:17:27,437 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:29,438 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:31,439 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:33,439 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:35,440 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:37,441 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:39,442 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:41,442 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:42,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:17:42,689 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:17:43,443 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:44,130 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:17:45,444 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:47,445 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:49,446 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:51,446 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:53,447 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:55,448 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:57,449 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:17:58,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:17:58,059 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:17:59,450 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:01,450 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:03,451 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:05,452 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:07,452 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:09,453 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:11,454 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:13,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:18:13,283 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:18:13,454 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:14,210 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:18:16,456 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:18,456 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:20,457 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:22,458 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:24,459 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:26,460 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:28,461 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:28,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:18:28,594 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:18:30,462 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:32,463 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:34,464 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:36,465 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:38,466 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:40,467 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:42,468 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:43,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:18:43,814 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:18:44,290 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:18:44,468 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:46,469 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:48,470 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:50,471 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:52,472 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:54,473 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:56,474 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:58,475 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:18:59,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:18:59,182 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:19:00,475 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:02,476 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:04,477 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:06,478 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:08,478 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:10,479 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:12,480 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:14,369 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:19:14,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:19:14,410 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:19:14,481 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:16,482 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:18,482 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:20,483 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:22,484 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:24,485 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:26,486 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:28,487 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:29,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:19:29,718 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:19:30,488 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:32,488 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:34,489 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:37,490 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:39,491 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:41,492 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:43,493 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:44,449 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:19:44,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:19:44,936 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:19:45,493 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:47,494 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:49,495 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:51,496 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:53,497 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:55,498 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:57,499 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:19:59,500 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:00,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:20:00,226 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:20:01,500 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:03,501 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:05,502 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:07,503 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:09,504 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:11,505 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:13,505 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:14,528 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:20:15,506 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:15,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:20:15,560 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:20:17,507 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:19,508 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:21,508 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:23,509 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:25,510 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:27,511 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:29,512 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:30,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:20:30,832 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:20:31,512 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:33,513 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:35,514 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:37,515 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:39,515 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:41,516 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:43,517 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:44,607 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:20:45,517 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:46,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:20:46,117 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:20:47,519 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:49,519 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:51,520 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:53,521 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:55,522 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:57,523 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:20:59,524 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:01,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:21:01,342 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:21:01,524 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:03,525 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:05,526 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:07,527 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:10,529 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:12,530 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:14,531 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:14,688 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:21:16,532 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:16,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:21:16,568 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:21:18,533 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:20,534 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:22,535 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:24,536 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:26,537 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:28,538 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:30,539 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:31,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:21:31,868 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:21:32,540 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:34,541 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:36,542 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:38,543 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:40,544 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:42,545 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:44,546 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:44,768 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:21:46,547 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:47,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:21:47,184 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:21:48,548 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:50,549 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:52,550 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:54,551 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:56,552 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:21:58,553 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:00,554 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:02,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:22:02,411 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:22:02,554 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:04,555 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:06,556 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:08,557 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:10,558 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:12,559 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:14,560 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:14,849 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:22:16,561 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:17,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:22:17,753 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:22:19,562 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:21,563 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:23,564 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:25,565 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:27,566 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:29,567 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:31,568 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:32,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:22:32,974 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:22:33,568 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:35,569 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:37,570 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:39,571 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:41,572 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:43,573 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:44,931 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:22:45,574 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:47,575 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:48,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:22:48,259 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:22:49,576 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:51,577 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:53,578 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:55,579 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:57,580 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:22:59,580 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:01,581 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:03,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:23:03,543 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:23:03,582 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:05,583 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:07,584 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:09,585 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:11,585 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:13,586 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:15,010 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:23:15,587 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:17,588 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:18,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:23:18,791 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:23:19,588 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:21,589 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:23,590 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:25,591 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:27,591 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:29,592 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:31,593 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:33,593 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:34,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:23:34,088 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:23:35,594 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:37,595 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:39,595 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:41,596 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:43,597 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:45,090 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:23:45,598 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:47,599 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:49,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:23:49,311 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:23:49,600 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:51,601 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:54,602 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:56,603 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:23:58,604 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:00,605 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:02,606 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:04,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:24:04,600 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:24:04,607 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:06,608 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:08,609 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:10,610 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:12,611 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:14,612 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:15,170 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:24:16,613 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:18,614 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:19,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:24:19,984 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:24:20,615 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:22,616 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:24,617 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:26,618 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:28,619 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:30,620 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:32,622 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:34,623 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:35,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:24:35,208 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:24:36,623 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:38,624 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:40,625 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:42,626 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:44,627 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:45,249 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:24:46,628 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:48,629 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:50,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:24:50,519 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:24:50,630 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:52,631 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:54,632 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:56,632 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:24:58,633 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:00,634 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:02,635 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:04,636 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:05,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:25:05,825 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:25:06,637 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:08,637 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:10,639 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:12,639 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:14,640 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:15,328 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:25:16,641 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:18,642 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:20,643 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:21,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:25:21,141 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:25:23,644 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:25,645 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:27,646 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:29,646 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:31,647 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:33,648 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:35,649 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:36,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:25:36,379 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:25:37,650 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:39,651 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:41,652 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:43,652 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:45,408 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:25:45,653 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:47,654 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:49,655 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:51,656 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:51,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:25:51,710 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:25:53,657 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:55,658 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:57,659 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:25:59,660 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:01,661 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:03,662 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:05,662 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:06,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:26:06,994 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:26:07,663 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:09,664 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:11,665 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:13,666 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:15,487 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:26:15,667 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:17,668 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:19,669 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:21,670 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:22,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:26:22,216 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:26:23,671 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:25,671 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:27,672 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:29,673 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:31,674 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:33,675 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:35,676 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:37,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:26:37,502 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:26:37,677 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:39,677 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:41,678 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:43,679 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:45,567 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:26:45,680 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:47,681 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:49,682 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:51,683 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:52,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:26:52,811 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:26:53,684 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:56,685 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:26:58,686 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:00,687 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:02,688 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:04,689 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:06,690 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:08,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:27:08,077 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:27:08,690 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:10,692 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:12,692 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:14,693 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:15,647 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:27:16,694 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:18,695 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:20,696 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:22,697 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:23,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:27:23,361 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:27:24,697 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:26,698 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:28,699 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:30,700 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:32,701 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:34,702 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:36,703 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:38,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:27:38,582 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:27:38,704 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:40,705 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:42,706 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:44,707 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:45,726 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:27:46,708 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:48,709 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:51,710 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:53,711 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:53,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:27:53,881 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:27:55,712 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:57,713 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:27:59,713 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:01,714 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:03,715 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:05,716 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:07,717 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:09,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:28:09,216 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:28:09,718 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:11,719 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:13,720 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:15,721 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:15,806 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:28:17,722 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:19,722 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:21,723 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:23,724 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:24,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:28:24,444 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:28:25,725 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:27,726 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:29,727 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:31,728 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:33,728 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:35,729 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:38,730 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:39,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:28:39,711 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:28:40,731 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:42,732 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:44,733 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:45,885 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:28:46,734 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:48,735 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:50,736 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:52,737 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:54,738 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:54,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:28:54,935 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:28:56,739 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:28:58,740 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:00,741 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:02,741 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:04,742 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:06,743 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:08,744 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:10,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:29:10,258 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:29:10,745 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:12,746 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:14,747 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:15,965 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:29:16,748 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:18,749 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:20,750 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:22,751 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:24,751 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:25,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:29:25,657 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:29:26,752 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:28,753 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:30,754 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:32,755 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:34,756 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:36,757 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:38,758 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:40,758 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:41,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:29:41,005 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:29:42,760 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:44,760 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:46,047 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:29:46,761 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:48,762 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:50,763 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:52,764 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:55,765 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:56,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:29:56,232 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:29:57,766 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:29:59,767 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:01,768 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:03,768 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:05,769 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:07,770 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:09,771 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:11,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:30:11,613 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:30:11,772 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:13,773 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:15,774 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:16,127 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:30:17,775 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:19,776 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:21,776 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:23,777 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:25,778 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:26,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:30:26,846 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:30:27,779 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:29,780 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:31,781 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:33,782 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:35,782 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:37,783 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:39,784 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:41,785 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:42,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:30:42,163 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:30:43,786 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:45,787 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:46,207 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:30:47,788 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:49,788 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:52,789 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:54,790 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:56,791 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:30:57,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:30:57,457 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:30:58,792 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:00,793 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:02,794 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:04,795 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:06,796 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:08,797 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:10,798 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:12,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:31:12,728 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:31:12,798 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:14,799 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:16,287 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:31:16,800 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:18,801 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:20,802 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:22,803 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:24,804 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:26,805 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:28,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:31:28,020 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:31:28,806 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:30,807 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:32,808 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:34,809 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:36,810 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:38,810 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:40,811 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:42,812 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:43,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:31:43,337 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:31:44,813 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:46,367 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:31:46,814 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:48,815 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:50,816 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:52,817 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:54,818 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:56,819 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:31:58,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:31:58,573 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:31:58,819 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:00,820 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:02,821 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:04,822 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:06,823 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:08,824 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:10,825 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:12,825 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:13,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:32:13,857 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:32:14,826 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:16,446 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:32:16,827 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:18,828 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:20,829 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:22,830 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:24,831 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:26,832 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:28,832 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:29,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:32:29,119 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:32:30,833 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:32,834 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:34,835 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:37,836 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:39,837 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:41,838 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:43,840 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:44,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:32:44,348 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:32:45,841 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:46,524 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:32:47,841 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:49,842 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:51,843 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:53,844 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:55,845 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:57,846 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:32:59,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:32:59,655 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:32:59,847 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:01,848 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:03,848 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:05,849 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:07,850 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:09,851 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:11,852 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:13,853 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:14,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:33:14,937 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:33:15,854 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:16,606 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:33:17,855 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:19,856 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:21,857 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:23,858 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:25,859 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:27,860 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:29,861 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:30,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:33:30,261 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:33:31,862 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:33,863 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:35,864 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:37,864 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:39,865 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:41,866 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:43,867 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:45,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:33:45,487 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:33:45,868 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:46,686 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:33:47,869 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:50,871 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:52,872 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:54,872 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:56,873 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:33:58,874 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:00,875 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:00,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:34:00,882 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:34:02,876 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:04,876 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:06,877 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:08,878 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:10,879 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:12,880 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:14,881 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:16,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:34:16,111 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:34:16,766 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:34:16,882 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:18,883 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:20,884 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:22,885 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:24,886 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:26,887 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:28,887 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:30,888 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:31,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:34:31,459 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:34:32,889 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:34,890 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:36,890 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:38,891 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:40,892 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:42,893 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:44,894 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:46,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:34:46,760 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:34:46,895 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:46,984 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:34:48,896 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:50,897 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:52,898 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:54,899 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:56,899 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:34:58,900 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:00,901 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:01,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:35:01,986 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:35:02,902 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:04,903 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:06,904 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:08,905 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:10,906 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:12,907 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:14,907 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:16,908 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:16,927 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:35:17,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:35:17,364 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:35:18,909 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:20,910 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:22,911 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:24,912 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:27,913 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:29,914 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:31,915 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:32,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:35:32,663 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:35:33,916 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:35,917 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:37,918 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:39,918 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:41,919 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:43,920 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:45,921 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:47,009 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:35:47,922 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:47,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:35:47,935 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:35:49,923 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:51,924 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:53,925 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:55,926 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:57,927 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:35:59,928 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:01,929 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:03,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:36:03,189 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:36:03,930 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:05,931 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:07,932 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:09,933 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:11,934 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:13,935 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:15,936 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:17,089 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:36:17,937 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:18,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:36:18,479 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:36:19,938 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:21,939 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:23,940 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:25,941 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:27,942 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:29,942 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:31,943 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:33,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:36:33,722 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:36:33,945 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:35,946 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:37,947 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:39,948 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:41,949 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:43,950 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:45,951 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:47,169 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:36:47,952 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:48,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:36:48,949 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:36:49,953 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:51,953 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:53,955 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:55,956 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:57,956 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:36:59,957 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:01,958 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:03,959 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:04,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:37:04,261 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:37:05,960 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:07,961 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:09,962 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:12,963 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:14,964 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:16,965 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:17,249 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:37:18,966 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:19,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:37:19,556 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:37:20,967 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:22,968 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:24,969 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:26,970 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:28,971 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:30,972 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:32,973 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:34,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:37:34,787 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:37:34,974 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:36,974 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:38,976 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:40,976 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:42,977 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:44,978 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:46,979 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:47,328 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:37:48,980 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:50,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:37:50,123 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:37:50,981 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:52,982 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:54,983 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:56,984 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:37:58,985 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:00,986 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:02,986 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:04,987 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:05,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:38:05,480 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:38:06,988 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:08,989 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:10,990 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:12,991 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:14,992 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:16,993 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:17,408 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:38:19,994 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:20,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:38:20,705 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:38:21,995 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:23,996 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:25,997 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:27,998 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:29,999 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:32,000 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:34,001 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:35,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:38:35,989 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:38:36,002 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:38,003 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:40,004 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:42,005 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:44,006 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:46,006 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:47,487 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:38:48,007 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:50,008 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:51,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:38:51,289 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:38:52,009 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:54,010 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:56,011 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:38:58,012 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:00,012 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:02,013 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:04,014 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:06,015 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:06,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:39:06,515 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:39:08,015 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:10,016 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:12,017 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:14,018 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:16,018 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:17,567 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:39:18,019 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:20,020 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:21,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:39:21,813 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:39:22,020 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:24,021 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:26,022 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:28,023 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:30,023 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:33,024 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:35,025 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:37,026 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:37,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:39:37,128 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:39:39,027 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:41,028 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:43,029 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:45,030 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:47,030 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:47,647 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:39:49,031 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:51,032 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:52,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:39:52,353 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:39:53,033 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:55,034 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:57,036 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:39:59,037 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:01,037 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:03,038 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:05,039 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:07,040 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:07,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:40:07,649 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:40:09,041 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:11,042 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:13,043 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:15,044 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:17,045 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:17,727 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:40:19,046 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:21,047 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:22,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:40:22,938 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:40:23,048 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:25,049 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:27,050 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:29,051 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:31,052 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:33,053 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:35,054 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:37,055 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:38,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:40:38,228 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:40:39,056 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:41,057 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:43,058 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:45,060 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:47,061 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:47,809 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:40:49,062 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:51,063 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:53,064 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:53,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:40:53,558 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:40:55,064 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:57,065 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:40:59,066 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:01,067 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:03,068 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:05,068 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:08,069 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:08,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:41:08,874 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:41:10,070 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:12,071 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:14,072 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:16,073 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:17,889 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:41:18,074 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:20,075 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:22,076 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:24,077 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:24,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:41:24,146 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:41:26,078 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:28,079 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:30,080 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:32,081 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:34,082 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:36,083 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:38,084 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:39,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:41:39,363 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:41:40,085 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:42,086 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:44,087 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:46,088 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:47,969 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:41:48,089 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:50,090 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:52,091 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:54,092 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:54,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:41:54,654 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:41:56,093 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:41:58,094 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:00,095 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:02,096 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:04,097 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:07,099 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:09,100 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:09,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:42:09,971 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:42:11,101 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:13,102 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:15,103 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:17,104 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:18,050 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:42:19,105 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:21,106 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:23,107 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:25,108 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:25,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:42:25,254 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:42:27,109 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:29,110 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:31,111 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:33,112 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:35,113 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:37,114 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:39,115 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:40,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:42:40,490 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:42:41,116 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:43,117 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:45,118 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:47,119 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:48,129 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:42:49,120 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:51,121 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:53,122 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:55,123 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:42:55,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:42:55,702 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:42:57,124 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:00,126 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:02,127 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:04,128 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:06,129 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:08,130 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:10,131 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:10,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:43:10,995 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:43:12,132 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:14,133 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:16,134 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:18,135 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:18,210 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:43:20,136 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:22,137 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:24,138 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:26,139 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:26,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:43:26,220 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:43:28,140 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:30,141 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:32,142 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:34,143 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:36,144 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:38,145 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:40,145 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:41,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:43:41,481 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:43:42,147 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:44,148 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:46,149 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:48,150 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:48,291 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:43:50,151 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:52,152 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:54,152 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:56,153 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:43:56,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:43:56,777 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:43:59,155 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:01,156 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:03,157 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:05,158 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:07,159 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:09,160 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:11,161 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:12,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:44:12,000 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:44:13,162 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:15,163 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:17,163 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:18,371 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:44:19,164 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:21,165 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:23,166 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:25,167 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:27,168 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:27,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:44:27,402 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:44:29,169 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:31,170 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:33,171 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:35,172 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:37,173 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:39,174 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:41,175 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:42,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:44:42,626 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:44:43,176 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:45,177 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:47,178 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:48,451 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:44:49,179 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:51,180 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:53,181 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:55,182 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:57,183 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:44:57,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:44:57,851 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:44:59,184 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:01,185 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:03,186 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:05,186 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:07,188 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:09,188 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:11,189 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:13,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:45:13,099 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:45:13,190 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:15,191 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:17,192 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:18,531 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:45:19,193 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:21,194 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:23,195 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:25,196 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:27,197 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:28,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:45:28,322 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:45:29,198 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:31,199 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:33,200 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:35,201 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:37,202 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:39,203 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:41,204 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:43,205 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:43,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:45:43,635 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:45:45,206 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:47,207 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:48,611 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:45:49,208 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:51,208 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:53,209 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:55,210 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:57,211 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:45:58,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:45:58,858 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:45:59,212 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:02,214 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:04,215 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:06,216 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:08,217 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:10,218 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:12,219 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:14,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:46:14,196 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:46:14,220 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:16,221 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:18,222 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:18,691 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:46:20,223 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:22,224 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:24,225 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:26,226 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:28,227 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:29,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:46:29,450 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:46:30,228 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:32,229 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:34,230 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:36,231 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:38,232 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:40,233 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:42,234 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:44,235 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:44,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:46:44,747 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:46:46,236 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:48,237 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:48,773 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:46:50,238 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:52,239 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:54,240 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:56,241 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:46:58,242 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:00,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:47:00,094 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:47:00,243 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:02,244 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:04,245 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:06,246 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:08,247 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:10,248 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:12,249 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:14,250 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:15,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:47:15,330 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:47:16,251 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:18,252 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:18,852 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:47:20,253 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:22,254 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:25,256 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:27,257 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:29,258 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:30,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:47:30,610 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:47:31,259 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:33,260 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:35,261 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:37,262 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:39,263 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:41,264 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:43,265 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:45,266 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:45,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:47:45,850 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:47:47,267 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:48,931 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:47:49,268 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:51,269 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:53,270 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:55,271 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:57,272 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:47:59,273 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:01,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:48:01,200 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:48:01,274 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:03,275 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:05,276 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:07,277 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:09,278 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:11,279 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:13,280 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:15,281 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:16,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:48:16,676 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:48:17,281 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:19,009 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:48:19,282 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:21,283 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:23,284 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:25,285 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:27,286 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:29,287 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:31,288 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:32,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:48:32,001 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:48:33,289 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:35,290 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:37,291 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:39,292 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:41,293 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:43,294 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:45,295 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:47,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:48:47,226 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:48:48,297 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:49,087 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:48:50,298 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:52,299 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:54,300 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:56,301 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:48:58,302 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:00,303 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:02,304 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:02,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:49:02,564 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:49:04,305 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:06,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:08,307 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:10,308 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:12,309 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:14,310 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:16,311 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:17,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:49:17,844 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:49:18,312 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:19,166 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:49:20,313 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:22,314 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:24,315 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:26,316 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:28,317 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:30,318 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:32,319 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:33,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:49:33,167 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:49:34,320 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:36,321 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:38,322 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:40,323 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:42,325 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:44,326 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:46,327 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:48,328 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:48,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:49:48,426 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:49:49,246 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:49:50,329 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:52,330 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:54,331 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:56,332 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:49:58,333 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:00,334 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:02,335 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:03,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:50:03,652 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:50:04,336 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:06,337 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:08,338 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:10,339 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:12,340 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:14,341 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:16,342 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:18,343 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:19,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:50:19,064 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:50:19,325 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:50:20,344 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:22,345 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:24,346 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:26,347 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:28,348 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:30,349 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:32,350 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:34,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:50:34,365 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:50:35,351 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:37,352 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:39,353 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:41,354 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:43,355 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:45,356 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:47,357 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:49,358 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:49,404 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:50:49,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:50:49,598 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:50:51,359 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:53,360 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:55,361 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:57,362 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:50:59,363 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:01,364 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:03,365 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:04,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:51:04,850 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:51:05,366 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:07,367 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:09,368 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:11,369 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:13,370 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:15,371 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:17,372 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:19,373 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:19,483 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:51:20,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:51:20,142 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:51:21,374 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:23,375 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:25,376 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:27,377 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:29,377 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:31,378 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:33,379 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:35,380 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:35,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:51:35,393 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:51:37,381 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:39,382 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:41,383 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:43,384 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:45,385 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:47,386 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:49,387 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:49,562 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:51:50,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:51:50,658 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:51:51,387 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:53,388 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:55,389 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:51:57,390 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:00,391 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:02,392 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:04,393 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:05,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:52:05,879 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:52:06,394 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:08,395 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:10,395 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:12,396 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:14,397 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:16,398 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:18,399 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:19,642 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:52:20,400 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:21,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:52:21,100 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:52:22,401 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:24,401 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:26,402 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:28,403 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:30,404 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:32,404 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:34,405 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:36,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:52:36,322 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:52:36,406 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:38,407 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:40,407 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:42,408 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:44,409 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:46,410 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:48,410 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:49,721 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:52:50,411 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:51,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:52:51,625 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:52:52,412 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:54,413 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:56,414 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:52:58,414 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:01,416 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:03,416 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:05,417 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:06,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:53:06,902 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:53:07,418 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:09,419 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:11,420 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:13,420 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:15,421 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:17,422 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:19,423 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:19,800 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:53:21,423 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:22,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:53:22,205 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:53:23,426 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:25,427 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:27,427 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:29,428 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:31,429 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:33,430 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:35,431 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:37,431 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:37,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:53:37,625 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:53:39,432 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:41,433 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:43,434 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:45,435 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:47,436 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:49,436 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:49,882 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:53:51,437 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:52,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:53:52,894 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:53:53,438 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:55,439 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:57,439 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:53:59,440 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:01,441 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:03,442 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:05,442 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:07,443 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:08,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:54:08,168 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:54:09,444 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:11,445 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:13,445 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:15,446 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:17,447 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:19,448 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:19,962 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:54:21,448 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:23,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:54:23,482 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:54:24,450 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:26,451 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:28,451 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:30,452 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:32,453 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:34,454 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:36,455 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:38,456 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:38,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:54:38,712 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:54:40,456 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:42,457 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:44,458 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:46,459 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:48,460 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:50,040 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:54:50,460 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:52,461 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:54,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:54:54,023 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:54:54,462 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:56,463 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:54:58,464 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:00,464 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:02,465 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:04,466 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:06,467 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:08,467 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:09,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:55:09,243 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:55:10,468 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:13,469 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:15,470 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:17,471 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:19,472 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:20,120 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:55:21,472 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:23,473 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:24,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:55:24,468 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:55:25,474 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:27,475 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:29,475 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:31,476 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:33,477 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:35,478 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:37,478 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:39,479 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:39,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:55:39,771 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:55:41,480 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:43,480 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:45,481 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:47,482 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:49,483 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:50,199 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:55:51,483 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:53,484 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:55,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:55:55,082 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:55:55,485 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:57,486 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:55:59,486 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:01,487 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:03,488 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:05,489 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:07,489 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:09,490 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:10,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:56:10,296 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:56:11,491 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:13,491 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:15,492 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:17,493 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:20,278 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:56:20,494 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:22,495 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:24,496 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:25,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:56:25,699 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:56:26,497 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:28,498 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:30,498 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:32,499 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:34,500 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:36,501 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:38,502 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:40,502 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:40,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:56:40,927 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:56:42,503 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:44,504 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:46,505 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:48,505 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:50,357 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:56:50,506 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:52,507 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:54,508 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:56,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:56:56,258 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:56:56,508 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:56:58,509 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:57:00,510 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:57:02,511 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:57:04,511 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:57:06,512 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:57:08,513 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:57:10,514 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:57:11,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:57:11,481 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:57:12,514 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:57:14,515 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:57:16,516 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:57:19,517 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:57:20,436 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:57:21,518 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:57:26,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:57:26,814 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:57:42,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:57:42,040 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:57:50,515 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:57:57,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:57:57,280 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:58:12,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:58:12,549 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:58:20,594 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:58:27,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:58:27,816 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:58:35,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 14:58:35,585 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 14:58:35,636 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 14:58:35,710 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 14:58:36,546 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 14:58:37,547 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:58:43,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:58:43,061 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:58:43,550 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:58:50,679 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:58:58,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:58:58,289 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:58:59,558 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 14:59:13,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:59:13,515 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:59:20,763 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:59:28,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:59:28,738 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:59:43,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:59:43,965 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 14:59:50,846 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 14:59:59,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 14:59:59,381 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:00:14,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:00:14,615 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:00:20,932 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:00:29,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:00:29,843 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:00:45,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:00:45,071 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:00:51,016 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:01:00,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:01:00,300 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:01:15,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:01:15,526 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:01:21,102 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:01:30,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:01:30,752 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:01:45,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:01:45,975 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:01:51,187 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:02:01,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:02:01,206 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:02:16,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:02:16,430 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:02:21,273 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:02:31,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:02:31,659 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:02:46,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:02:46,885 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:02:51,358 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:03:02,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:03:02,139 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:03:03,681 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:03:05,682 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:03:05,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:05,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:05,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:05,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:05,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:05,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:05,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:05,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:05,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:05,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:05,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:05,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:05,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:05,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:05,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:05,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:07,683 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:03:08,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:08,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:08,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:08,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:08,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:08,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:08,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:08,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:08,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:08,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:08,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:08,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:08,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:08,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:08,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:08,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,683 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:03:09,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:09,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:11,684 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:03:11,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:11,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:11,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:11,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:11,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:11,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:11,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:11,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:11,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:11,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:11,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:11,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:11,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:11,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:11,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:11,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,685 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:03:13,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:13,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:14,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,685 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:03:15,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:15,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:16,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:16,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:16,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:16,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:16,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:16,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:16,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:16,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:16,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:16,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:16,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:16,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:16,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:16,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:16,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:16,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:03:17,368 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:03:17,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,687 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:03:17,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:17,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:18,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:19,688 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:03:20,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:20,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:21,440 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:03:21,689 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:03:22,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:22,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,689 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:03:23,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:23,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:24,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:24,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:24,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:24,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:24,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:24,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:24,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:24,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:24,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:24,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:24,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:24,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:24,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:24,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:24,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:24,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:25,690 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:03:26,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:26,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:27,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:27,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:27,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:27,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:27,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:27,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:27,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:27,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:27,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:27,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:27,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:27,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:27,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:27,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:27,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:27,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:27,691 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:03:28,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:28,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:29,692 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:03:30,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:30,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:30,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:30,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:30,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:30,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:30,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:30,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:30,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:30,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:30,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:30,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:30,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:30,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:30,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:30,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:31,692 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:03:32,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:03:32,621 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:03:33,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:33,693 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:03:34,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:34,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:35,694 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:03:35,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:35,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:36,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:36,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:36,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:36,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:36,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:36,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:36,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:36,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:36,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:36,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:36,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:36,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:36,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:36,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:37,695 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:03:38,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:38,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:38,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:38,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:38,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:38,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:38,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:38,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:38,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:38,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:38,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:38,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:38,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:38,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:38,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:38,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:03:39,695 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:03:41,696 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:03:43,697 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:03:45,698 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:03:47,698 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:03:47,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:03:47,859 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:03:51,516 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:03:51,700 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:03:55,701 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:03:59,703 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:04:03,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:04:03,101 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:04:03,705 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:04:08,707 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:04:10,708 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:04:14,710 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:04:18,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:04:18,322 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:04:18,712 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:04:20,713 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:04:21,591 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:04:24,715 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:04:26,716 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:04:28,717 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:04:32,718 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:04:33,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:04:33,549 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:04:34,720 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:04:36,720 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:04:40,722 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:04:42,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:42,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:42,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:42,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:42,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:42,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:42,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:42,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:42,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:42,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:42,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:42,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:42,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:42,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:42,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:42,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:42,723 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:04:44,724 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:04:46,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:46,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:46,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:46,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:46,725 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:04:47,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:47,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:47,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:47,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:47,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:47,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:47,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:47,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:48,726 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:04:48,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:04:48,780 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:04:50,727 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:04:51,667 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:04:54,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:54,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:54,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:54,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:54,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:54,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:54,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:54,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:54,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:54,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:54,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:54,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:54,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:54,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:54,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:54,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:54,729 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:04:56,730 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:04:58,731 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:04:58,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:04:58,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:00,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:00,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:00,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:00,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:00,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:00,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:00,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:00,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:00,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:00,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:00,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:00,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:00,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:00,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:00,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:00,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:00,732 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:05:01,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:01,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:02,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:02,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:02,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:02,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:02,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:02,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:02,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:02,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:02,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:02,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:02,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:02,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:02,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:02,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:02,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:02,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:02,733 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:05:03,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:03,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:04,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:05:04,061 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:05:04,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:04,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:04,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:04,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:04,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:04,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:04,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:04,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:04,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:04,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:04,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:04,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:04,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:04,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:04,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:04,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:04,734 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:05:06,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:06,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:06,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:06,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:06,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:06,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:06,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:06,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:06,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:06,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:06,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:06,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:06,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:06,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:06,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:06,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:06,735 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:05:07,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:07,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:08,736 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:05:08,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:08,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:08,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:08,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:08,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:08,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:08,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:08,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:08,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:08,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:08,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:08,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:08,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:08,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:08,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:08,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:09,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:10,737 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:05:11,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:11,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:12,738 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:05:13,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:13,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:14,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:14,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:14,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:14,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:14,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:14,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:14,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:14,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:14,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:14,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:14,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:14,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:14,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:14,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:14,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:14,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:14,739 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:05:15,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:15,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:16,740 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:05:17,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:17,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:18,741 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:05:18,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:18,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:18,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:18,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:18,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:18,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:18,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:18,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:18,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:18,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:18,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:18,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:18,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:18,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:18,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:18,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:19,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:05:19,290 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:05:19,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:19,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:19,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:19,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:19,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:19,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:19,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:19,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:19,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:19,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:19,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:19,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:19,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:19,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:19,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:19,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:20,742 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:05:21,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:21,745 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:05:22,743 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:05:23,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:23,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:24,744 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:05:24,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:24,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:24,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:24,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:24,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:24,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:24,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:24,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:24,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:24,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:24,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:24,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:24,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:24,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:24,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:24,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:05:26,745 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:05:28,746 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:05:30,747 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:05:32,747 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:05:34,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:05:34,518 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:05:34,748 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:05:38,750 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:05:42,752 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:05:46,754 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:05:49,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:05:49,864 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:05:50,755 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:05:51,821 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:05:54,756 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:05:56,757 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:00,758 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:04,759 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:05,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:06:05,085 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:06:06,760 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:10,761 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:12,762 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:16,764 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:18,764 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:20,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:06:20,310 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:06:20,765 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:21,901 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:06:22,766 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:26,767 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:28,768 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:30,769 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:32,770 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:33,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:33,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:33,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:33,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:33,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:33,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:33,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:33,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:33,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:33,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:33,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:33,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:34,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:34,102 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 15:06:34,340 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 15:06:34,418 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 15:06:34,771 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:34,771 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 15:06:34,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:34,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:34,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:34,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:34,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:34,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:34,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:34,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:34,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:34,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:34,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:34,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:34,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:34,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:34,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:34,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:35,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:06:35,535 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:06:36,772 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:38,773 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:41,774 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:43,775 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:45,776 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:46,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:46,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:46,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:46,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:46,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:46,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:46,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:46,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:46,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:46,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:46,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:46,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:46,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:46,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:46,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:46,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:47,777 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:48,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:48,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:48,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:48,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:48,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:48,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:48,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:48,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:48,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:48,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:48,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:48,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:48,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:48,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:48,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:48,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:49,778 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:50,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:06:50,762 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:06:51,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:51,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:51,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:51,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:51,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:51,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:51,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:51,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:51,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:51,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:51,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:51,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:51,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:51,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:51,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:51,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:51,779 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:52,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,068 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:06:52,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:52,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:53,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:53,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:53,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:53,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:53,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:53,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:53,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:53,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:53,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:53,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:53,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:53,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:53,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:53,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:53,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:53,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:53,780 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:54,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:54,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:55,781 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:57,782 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:06:57,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:57,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:57,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:57,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:57,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:57,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:57,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:57,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:57,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:57,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:57,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:57,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:57,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:57,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:57,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:57,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:58,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:59,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:59,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:59,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:59,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:59,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:59,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:59,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:59,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:59,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:59,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:59,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:59,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:59,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:59,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:59,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:59,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:06:59,783 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:07:00,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:00,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:00,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:00,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:00,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:00,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:00,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:00,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:00,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:00,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:00,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:00,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:00,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:00,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:00,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:00,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,784 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:07:01,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:01,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:02,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:03,786 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:07:04,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:04,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:05,787 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:07:05,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:07:05,991 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:07:06,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:06,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:07,788 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:07:08,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:08,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:09,789 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:07:10,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:10,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:11,790 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:07:12,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:12,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:12,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:12,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:12,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:12,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:12,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:12,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:12,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:12,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:12,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:12,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:12,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:12,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:12,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:12,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:13,791 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:07:14,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:14,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:07:15,792 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:07:17,793 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:07:19,794 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:07:21,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:07:21,224 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:07:21,794 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:07:22,153 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:07:25,796 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:07:31,800 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:07:33,800 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:07:36,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:07:36,488 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:07:37,802 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:07:41,804 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:07:45,806 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:07:47,807 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:07:51,809 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:07:51,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:07:51,885 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:07:52,232 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:07:55,811 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:07:57,811 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:07:59,812 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:03,814 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:05,815 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:07,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:08:07,163 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:08:07,816 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:11,818 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:13,819 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:15,820 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:17,820 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:20,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:20,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:20,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:20,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:21,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:21,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:21,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:21,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:21,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:21,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:21,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:21,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:21,822 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:22,307 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:08:22,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:08:22,400 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:08:23,823 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:25,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:25,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:25,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:25,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:25,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:25,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:25,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:25,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:25,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:25,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:25,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:25,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:25,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:25,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:25,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:25,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:25,824 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:27,825 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:29,825 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:31,826 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:33,827 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:34,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:34,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:34,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:34,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:34,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:34,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:34,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:34,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:34,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:34,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:34,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:34,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:34,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:34,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:34,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:34,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:35,828 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:37,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:08:37,714 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:08:37,829 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:39,830 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:40,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:40,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:40,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:40,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:40,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:40,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:40,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:40,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:40,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:40,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:40,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:40,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:40,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:40,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:40,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:40,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:41,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:41,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:41,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:41,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:41,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:41,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:41,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:41,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:41,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:41,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:41,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:41,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:41,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:41,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:41,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:41,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:41,831 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:42,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:42,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:43,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:43,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:43,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:43,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:43,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:43,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:43,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:43,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:43,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:43,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:43,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:43,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:43,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:43,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:43,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:43,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:43,832 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:44,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:44,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:44,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:44,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:44,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:44,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:44,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:44,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:44,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:44,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:44,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:44,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:44,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:44,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:44,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:44,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,833 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:45,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:45,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:46,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:47,834 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:48,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:48,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,834 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:49,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:49,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:50,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:50,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:50,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:50,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:50,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:50,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:50,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:50,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:50,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:50,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:50,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:50,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:50,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:50,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:50,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:50,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,835 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:51,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:51,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,383 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:08:52,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:52,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:53,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:08:53,081 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:08:53,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:53,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:53,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:53,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:53,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:53,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:53,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:53,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:53,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:53,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:53,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:53,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:53,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:53,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:53,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:53,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:53,837 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:54,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:54,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:55,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:55,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:55,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:55,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:55,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:55,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:55,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:55,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:55,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:55,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:55,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:55,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:55,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:55,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:55,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:55,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:56,838 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:57,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:57,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:57,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:57,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:57,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:57,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:57,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:57,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:57,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:57,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:57,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:57,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:57,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:57,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:57,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:57,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:58,839 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:08:59,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:08:59,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:00,840 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:09:02,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:02,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:02,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:02,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:02,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:02,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:02,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:02,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:02,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:02,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:02,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:02,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:02,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:02,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:02,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:02,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:02,840 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:09:03,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:03,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:03,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:03,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:03,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:03,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:03,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:03,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:03,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:03,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:03,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:03,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:03,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:03,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:03,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:03,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:04,841 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:09:06,842 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:09:08,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:09:08,302 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:09:08,843 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:09:12,845 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:09:15,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:15,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:15,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:15,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:15,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:15,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:15,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:15,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:15,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:15,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:15,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:15,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:15,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:15,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:15,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:15,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:09:16,847 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:09:20,848 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:09:22,472 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:09:23,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:09:23,554 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:09:24,850 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:09:28,852 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:09:30,853 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:09:34,855 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:09:36,856 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:09:38,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:09:38,785 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:09:40,857 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:09:42,858 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:09:46,860 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:09:48,861 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:09:50,862 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:09:52,548 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:09:54,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:09:54,115 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:09:54,864 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:09:56,864 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:09:58,865 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:00,866 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:02,867 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:06,869 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:06,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:06,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:06,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:06,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:07,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:07,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:07,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:07,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:07,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:07,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:07,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:07,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:07,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:07,599 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 15:10:07,819 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 15:10:07,903 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 15:10:08,895 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:08,896 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 15:10:09,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:10:09,342 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:10:09,896 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:10,896 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:12,897 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:14,898 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:15,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:15,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:15,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:15,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:15,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:15,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:15,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:15,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:15,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:15,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:15,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:15,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:15,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:15,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:15,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:15,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:16,899 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:18,900 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:20,901 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:22,637 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:10:22,901 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:23,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:23,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:23,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:23,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:23,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:23,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:23,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:23,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:23,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:23,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:23,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:23,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:23,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:23,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:23,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:23,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:24,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:10:24,573 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:10:24,902 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:26,903 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:28,905 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:30,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,905 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:30,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:30,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:31,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:31,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:31,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:31,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:31,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:31,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:31,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:31,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:31,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:31,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:31,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:31,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:31,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:31,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:31,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:31,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:32,906 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:33,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:33,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:34,907 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:35,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:35,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,908 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:36,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:36,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:38,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:38,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:38,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:38,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:38,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:38,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:38,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:38,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:38,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:38,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:38,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:38,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:38,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:38,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:38,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:38,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:38,909 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:39,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:39,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:10:39,798 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:10:40,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:40,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:41,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:41,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:41,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:41,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:41,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:41,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:41,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:41,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:41,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:41,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:41,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:41,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:41,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:41,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:41,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:41,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:41,911 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:42,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:42,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:43,911 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:45,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:45,912 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:46,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:46,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,913 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:47,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:47,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:48,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:49,914 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:50,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:50,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:50,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:50,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:50,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:50,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:50,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:50,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:50,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:50,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:50,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:50,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:50,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:50,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:50,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:50,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:51,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:51,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:51,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:51,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:51,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:51,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:51,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:51,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:51,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:51,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:51,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:51,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:51,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:51,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:51,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:51,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:51,915 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:52,717 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:10:53,916 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:10:55,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:10:55,130 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:10:59,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:59,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:59,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:59,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:59,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:59,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:59,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:59,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:59,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:59,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:59,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:59,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:59,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:59,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:59,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:59,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:10:59,918 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:11:03,920 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:11:07,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:07,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:07,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:07,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:07,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:07,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:07,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:07,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:07,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:07,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:07,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:07,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:07,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:07,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:07,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:07,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:07,922 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:11:10,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:11:10,462 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:11:11,924 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:11:15,925 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:11:17,926 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:11:21,928 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:11:22,796 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:11:25,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:11:25,688 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:11:25,930 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:11:27,930 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:11:31,932 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:11:33,933 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:11:35,934 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:11:39,936 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:11:41,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:11:41,079 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:11:41,937 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:11:43,938 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:11:45,939 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:11:49,941 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:11:51,941 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:11:52,873 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:11:53,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:53,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:53,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:53,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:53,942 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:11:54,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:54,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:54,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:54,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:54,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:54,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:54,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:54,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:11:55,943 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:11:56,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:11:56,313 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:11:57,944 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:11:59,945 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:12:01,946 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:12:03,947 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:12:04,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:04,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:04,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:04,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:04,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:04,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:04,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:04,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:04,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:04,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:04,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:04,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:04,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:04,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:04,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:04,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:05,948 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:12:07,949 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:12:09,950 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:12:11,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:12:11,718 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:12:11,951 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:12:12,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:12,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:12,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:12,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:12,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:12,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:12,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:12,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:12,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:12,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:12,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:12,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:12,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:12,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:12,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:12,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:13,951 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:12:15,952 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:12:17,953 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:12:19,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,954 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:12:19,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:19,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:21,956 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:12:22,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:22,948 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:12:23,957 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:12:24,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:24,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:25,958 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:12:26,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:26,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:12:26,943 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:12:27,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:27,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:27,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:27,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:27,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:27,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:27,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:27,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:27,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:27,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:27,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:27,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:27,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:27,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:27,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:27,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:27,959 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:12:28,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:28,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:29,960 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:12:30,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:30,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:30,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:30,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:30,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:30,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:30,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:30,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:30,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:30,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:30,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:30,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:30,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:30,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:30,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:30,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,960 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:12:31,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:31,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:32,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:33,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:33,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:33,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:33,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:33,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:33,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:33,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:33,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:33,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:33,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:33,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:33,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:33,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:33,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:33,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:33,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:33,961 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:12:34,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:34,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:35,962 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:12:36,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:36,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:36,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:36,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:36,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:36,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:36,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:36,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:36,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:36,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:36,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:36,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:36,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:36,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:36,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:36,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:37,963 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:12:38,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:38,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:39,964 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:12:41,965 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:12:42,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:42,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:42,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:42,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:42,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:42,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:42,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:42,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:42,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:42,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:42,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:42,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:42,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:42,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:42,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:42,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:42,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:12:42,170 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:12:45,966 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:12:49,968 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:12:53,026 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:12:53,970 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:12:54,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:54,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:54,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:54,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:54,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:54,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:54,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:54,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:54,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:54,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:54,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:54,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:54,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:54,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:54,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:54,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:12:57,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:12:57,396 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:12:57,972 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:01,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:01,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:01,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:01,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:01,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:01,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:01,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:01,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:01,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:01,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:01,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:01,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:01,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:01,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:01,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:01,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:01,973 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:03,974 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:07,976 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:10,977 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:12,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:13:12,916 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:13:14,979 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:16,980 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:20,982 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:22,983 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:23,101 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:13:24,984 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:28,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:13:28,151 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:13:28,986 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:30,987 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:32,988 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:36,989 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:38,990 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:40,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:40,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:40,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:40,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:40,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:40,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:40,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:40,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:40,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:40,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:40,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:40,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:40,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:40,749 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 15:13:40,964 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 15:13:41,032 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:41,033 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 15:13:42,032 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:42,032 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 15:13:43,033 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:43,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:13:43,398 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:13:45,033 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:47,034 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:49,035 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:51,036 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:52,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:52,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:52,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:52,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:52,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:52,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:52,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:52,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:52,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:52,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:52,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:52,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:52,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:52,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:52,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:52,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:53,037 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:53,209 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:13:55,038 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:57,039 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:58,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:13:58,689 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:13:59,040 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:13:59,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:59,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:59,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:59,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:59,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:59,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:59,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:59,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:59,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:59,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:59,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:59,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:59,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:59,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:59,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:13:59,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:01,041 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:14:03,042 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:14:05,043 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:14:07,044 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:14:07,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:07,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:07,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:07,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:07,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:07,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:07,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:07,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:07,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:07,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:07,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:07,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:07,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:07,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:07,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:07,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:08,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:08,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:08,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:08,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:08,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:08,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:08,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:08,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:08,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:08,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:08,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:08,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:08,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:08,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:08,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:08,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:09,045 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:14:09,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:09,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:09,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:09,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:09,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:09,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:09,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:09,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:09,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:09,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:09,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:09,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:09,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:09,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:09,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:09,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,046 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:14:11,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:11,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:12,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,047 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:14:13,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:13,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:14:13,915 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:14:14,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:14,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:14,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:14,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:14,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:14,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:14,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:14,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:14,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:14,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:14,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:14,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:14,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:14,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:14,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:14,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,047 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:14:15,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:15,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:16,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,048 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:14:17,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:17,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:18,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:19,049 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:14:19,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:19,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:19,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:19,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:19,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:19,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:19,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:19,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:19,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:19,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:19,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:19,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:19,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:19,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:19,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:19,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:20,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,050 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:14:21,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:21,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:23,051 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:14:23,288 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:14:23,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:23,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:23,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:23,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:23,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:23,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:23,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:23,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:23,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:23,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:23,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:23,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:23,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:23,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:23,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:23,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:24,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,052 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:14:25,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:25,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:27,053 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:14:27,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:27,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:27,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:27,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:27,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:27,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:27,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:27,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:27,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:27,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:27,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:27,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:27,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:27,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:27,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:27,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:29,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:14:29,141 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:14:29,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:29,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:29,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:29,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:29,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:29,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:29,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:29,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:29,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:29,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:29,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:29,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:29,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:29,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:29,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:29,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:29,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:29,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:29,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:29,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:29,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:30,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:31,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:31,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:31,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:31,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:31,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:31,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:31,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:31,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:31,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:31,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:31,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:31,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:31,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:31,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:31,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:31,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:33,055 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:14:33,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:33,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:33,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:33,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:33,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:33,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:33,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:33,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:33,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:33,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:33,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:33,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:33,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:33,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:33,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:33,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:36,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:36,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:36,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:36,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:36,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:36,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:36,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:36,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:36,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:36,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:36,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:36,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:36,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:36,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:36,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:36,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:37,057 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:14:37,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:37,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:37,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:37,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:37,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:37,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:37,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:37,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:37,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:37,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:37,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:37,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:37,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:37,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:37,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:37,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:41,059 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:14:43,060 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:14:44,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:14:44,387 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:14:47,061 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:14:48,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:48,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:48,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:48,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:48,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:48,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:48,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:48,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:48,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:48,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:48,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:48,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:48,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:48,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:48,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:48,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:51,063 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:14:53,366 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:14:55,065 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:14:55,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:55,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:55,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:55,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:55,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:55,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:55,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:55,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:55,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:55,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:55,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:55,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:55,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:55,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:55,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:55,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:14:57,066 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:14:59,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:14:59,613 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:15:01,068 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:03,068 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:07,070 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:09,071 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:12,073 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:14,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:15:14,845 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:15:16,074 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:18,075 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:20,076 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:22,077 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:23,444 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:15:26,079 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:26,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:26,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:26,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:26,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:27,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:27,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:27,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:27,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:27,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:27,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:27,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:27,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:28,080 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:30,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:15:30,074 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:15:30,081 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:32,082 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:34,083 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:36,084 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:38,085 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:40,086 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:42,087 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:43,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:43,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:43,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:43,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:43,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:43,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:43,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:43,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:43,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:43,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:43,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:43,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:43,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:43,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:43,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:43,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:44,088 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:45,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:15:45,302 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:15:46,088 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:48,089 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:50,090 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:51,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:51,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:51,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:51,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:51,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:51,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:51,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:51,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:51,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:51,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:51,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:51,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:51,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:51,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:51,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:51,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:52,091 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:53,521 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:15:54,092 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:56,093 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:58,094 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:15:58,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:58,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:58,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:58,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:58,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:58,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:58,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:58,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:58,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:58,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:58,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:58,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:58,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:58,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:58,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:58,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:59,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:59,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:59,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:59,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:59,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:59,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:59,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:59,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:59,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:59,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:59,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:59,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:59,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:59,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:59,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:15:59,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,095 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:16:00,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:16:00,531 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:16:00,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:00,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:01,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,096 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:16:02,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:02,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:03,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:03,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:03,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:03,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:03,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:03,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:03,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:03,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:03,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:03,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:03,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:03,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:03,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:03,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:03,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:03,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:04,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:04,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:04,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:04,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:04,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:04,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:04,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:04,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:04,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:04,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:04,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:04,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:04,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:04,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:04,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:04,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:04,097 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:16:05,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:05,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,098 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:16:06,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:06,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:07,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:07,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:07,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:07,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:07,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:07,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:07,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:07,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:07,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:07,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:07,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:07,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:07,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:07,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:07,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:07,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,099 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:16:08,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:08,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:09,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,099 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:16:10,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:10,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:11,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:11,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:11,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:11,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:11,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:11,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:11,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:11,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:11,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:11,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:11,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:11,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:12,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:12,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:12,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:12,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:12,100 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:16:12,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:12,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:12,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:12,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:12,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:12,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:12,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:12,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:12,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:12,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:12,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:12,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:12,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:12,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:12,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:12,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:14,101 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:16:15,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:15,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:15,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:15,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:15,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:15,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:15,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:15,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:16:15,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:15,756 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:16:15,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:15,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:15,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:15,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:15,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:15,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:15,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:15,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,448 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:17,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,104 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:16:20,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:20,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:23,602 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:16:23,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:23,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:23,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:23,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:23,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:23,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:23,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:23,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:23,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:23,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:23,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:23,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:23,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:23,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:23,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:23,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:24,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:24,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:24,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:24,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:24,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:24,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:24,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:24,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:24,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:24,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:24,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:24,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:24,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:24,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:24,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:24,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:24,106 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:16:25,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:25,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:28,108 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:16:29,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:29,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:29,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:29,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:29,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:29,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:29,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:29,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:29,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:29,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:29,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:29,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:29,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:29,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:29,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:29,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:30,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:16:30,990 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:16:32,109 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:16:32,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:32,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:32,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:32,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:32,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:32,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:32,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:32,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:32,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:32,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:32,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:32,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:32,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:32,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:32,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:32,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:34,110 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:16:37,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:37,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:38,112 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:16:42,113 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:16:44,114 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:16:46,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:16:46,236 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:16:46,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:46,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:46,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:46,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:46,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:46,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:46,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:46,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:46,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:46,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:46,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:46,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:46,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:46,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:46,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:46,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:47,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:47,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:47,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:47,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:47,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:47,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:47,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:47,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:47,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:47,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:47,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:47,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:47,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:47,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:47,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:47,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:16:48,116 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:16:50,117 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:16:53,681 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:16:54,119 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:16:56,120 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:16:58,121 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:00,122 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:03,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:17:03,492 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:17:04,123 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:06,124 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:08,125 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:10,126 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:13,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:13,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:13,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:13,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:13,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:13,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:13,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:13,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:13,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:13,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:13,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:13,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:13,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:14,075 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 15:17:14,206 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:14,308 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 15:17:14,371 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 15:17:15,201 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:15,202 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 15:17:16,202 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:18,203 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:18,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:17:18,719 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:17:20,204 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:22,205 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:23,795 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:17:24,205 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:26,206 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:28,207 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:30,208 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:30,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:30,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:30,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:30,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:30,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:30,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:30,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:30,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:30,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:30,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:30,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:30,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:30,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:30,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:30,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:30,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:32,208 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:33,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:17:33,949 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:17:34,209 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:36,210 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:37,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:37,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:37,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:37,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:37,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:37,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:37,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:37,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:37,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:37,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:37,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:37,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:37,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:37,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:37,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:37,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:38,211 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:40,212 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:42,213 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:45,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:45,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:45,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:45,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:45,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:45,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:45,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:45,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:45,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:45,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:45,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:45,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:45,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:45,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:45,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:45,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:45,214 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:47,215 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:47,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:47,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:48,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:17:49,196 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:17:49,216 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:49,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:49,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:50,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,217 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:51,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:51,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:52,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:52,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:52,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:52,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:52,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:52,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:52,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:52,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:52,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:52,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:52,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:52,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:52,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:52,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:52,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:52,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,217 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:53,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:53,874 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:17:54,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:54,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:55,218 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:55,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:55,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:55,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:55,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:55,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:55,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:55,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:55,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:55,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:55,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:55,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:55,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:55,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:55,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:55,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:55,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:56,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:56,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:56,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:56,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:56,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:56,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:56,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:56,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:56,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:56,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:56,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:56,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:56,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:56,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:56,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:56,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,219 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:17:57,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:57,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:58,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:17:59,220 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:18:01,220 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:18:04,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:18:04,435 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:18:05,222 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:18:05,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:05,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:05,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:05,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:05,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:05,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:05,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:05,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:05,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:05,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:05,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:05,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:05,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:05,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:05,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:05,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:07,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:07,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:07,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:07,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:07,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:07,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:07,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:07,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:07,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:07,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:07,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:07,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:07,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:07,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:07,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:07,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:09,224 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:18:09,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:09,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:09,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:09,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:09,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:09,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:09,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:09,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:09,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:09,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:09,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:09,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:09,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:09,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:09,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:09,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,225 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:18:13,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:13,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:16,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:16,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:16,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:16,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:16,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:16,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:16,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:16,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:16,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:16,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:16,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:16,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:16,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:16,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:16,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:16,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:17,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:17,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:17,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:17,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:17,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:17,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:17,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:17,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:17,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:17,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:17,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:17,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:17,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:17,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:17,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:17,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:17,234 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:18:18,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:18,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:18,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:18,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:18,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:18,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:18,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:18,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:18,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:18,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:18,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:18,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:18,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:18,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:18,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:18,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:19,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:18:19,663 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:18:20,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:20,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:21,236 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:18:23,954 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:18:25,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:25,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:25,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:25,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:25,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:25,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:25,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:25,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:25,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:25,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:25,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:25,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:25,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:25,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:25,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:25,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:25,237 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:18:26,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:26,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:26,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:26,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:26,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:26,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:26,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:26,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:26,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:26,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:26,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:26,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:26,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:26,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:26,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:26,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:27,238 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:18:31,240 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:18:33,241 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:18:34,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:34,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:34,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:34,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:34,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:34,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:34,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:34,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:34,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:34,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:34,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:34,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:34,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:34,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:34,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:34,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:34,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:18:34,896 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:18:35,241 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:18:35,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:35,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:35,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:35,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:35,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:35,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:35,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:35,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:35,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:35,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:35,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:35,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:35,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:35,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:35,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:35,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:38,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:38,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:38,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:38,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:38,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:38,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:38,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:38,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:38,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:38,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:38,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:38,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:38,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:38,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:38,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:38,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:39,243 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:18:41,244 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:18:43,245 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:18:47,246 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:18:49,247 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:18:50,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:18:50,352 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:18:51,248 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:18:53,249 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:18:54,033 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:18:55,250 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:18:58,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:58,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:58,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:58,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:59,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:59,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:59,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:59,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:59,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:59,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:59,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:59,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:18:59,251 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:19:01,252 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:19:03,253 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:19:05,254 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:19:05,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:19:05,576 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:19:07,255 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:19:09,255 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:19:11,256 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:19:13,257 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:19:15,258 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:19:17,259 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:19:19,260 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:19:20,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:20,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:20,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:20,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:20,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:20,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:20,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:20,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:20,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:20,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:20,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:20,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:20,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:20,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:20,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:20,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:20,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:19:20,851 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:19:21,261 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:19:23,261 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:19:24,111 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:19:25,262 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:19:25,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:25,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:25,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:25,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:25,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:25,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:25,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:25,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:25,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:25,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:25,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:25,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:25,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:25,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:25,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:25,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:27,263 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:19:29,264 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:19:31,265 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:19:33,266 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:19:33,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:33,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:33,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:33,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:33,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:33,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:33,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:33,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:33,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:33,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:33,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:33,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:33,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:33,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:33,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:33,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:35,267 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:19:35,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:35,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:35,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:35,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:35,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:35,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:35,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:35,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:35,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:35,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:35,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:35,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:35,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:35,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:35,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:35,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:36,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:19:36,124 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:19:36,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:36,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:36,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:36,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:36,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:36,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:36,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:36,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:36,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:36,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:36,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:36,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:36,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:36,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:36,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:36,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,267 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:19:37,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:37,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:38,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:39,268 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:19:39,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:39,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:39,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:39,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:39,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:39,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:39,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:39,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:39,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:39,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:39,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:39,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:39,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:39,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:39,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:39,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:40,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:40,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:40,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:40,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:40,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:40,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:40,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:40,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:40,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:40,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:40,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:40,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:40,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:40,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:40,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:40,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,269 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:19:41,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:41,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:42,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:43,270 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:19:45,270 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:19:45,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:45,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:45,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:45,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:45,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:45,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:45,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:45,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:45,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:45,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:45,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:45,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:45,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:45,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:45,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:45,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:48,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:48,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:48,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:48,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:48,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:48,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:48,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:48,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:48,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:48,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:48,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:48,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:48,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:48,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:48,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:48,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:48,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:48,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:49,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:49,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:49,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:49,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:49,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:49,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:49,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:49,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:49,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:49,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:49,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:49,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:49,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:49,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:50,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:50,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:51,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:51,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:51,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:51,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:51,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:51,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:51,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:51,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:51,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:51,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:51,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:51,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:51,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:51,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:51,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:19:51,354 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:19:52,273 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:19:52,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:52,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:54,190 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:19:54,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:54,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:54,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:54,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:54,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:54,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:54,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:54,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:54,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:54,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:54,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:54,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:54,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:54,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:54,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:54,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:19:56,275 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:20:00,276 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:20:02,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:02,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,278 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:20:04,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:04,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:05,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:05,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:05,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:05,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:05,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:05,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:06,280 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:20:06,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:20:06,816 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:20:07,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:07,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:10,281 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:20:10,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:10,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:10,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:10,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:10,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:10,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:10,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:10,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:10,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:10,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:10,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:10,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:10,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:10,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:10,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:10,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:12,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:12,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:12,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:12,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:12,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:12,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:12,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:12,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:12,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:12,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:12,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:12,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:12,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:12,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:12,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:12,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:13,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:13,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:13,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:13,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:13,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:13,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:13,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:13,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:13,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:13,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:13,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:13,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:13,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:13,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:13,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:13,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:14,283 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:20:16,283 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:20:16,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:16,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:16,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:16,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:16,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:16,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:16,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:16,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:16,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:16,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:16,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:16,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:16,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:16,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:16,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:16,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:18,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:18,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:18,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:18,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:18,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:18,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:18,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:18,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:18,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:18,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:18,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:18,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:18,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:18,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:18,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:18,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:20,285 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:20:21,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:21,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:21,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:21,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:21,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:21,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:21,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:21,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:21,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:21,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:21,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:21,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:21,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:21,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:21,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:21,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:22,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:20:22,049 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:20:22,286 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:20:24,284 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:20:24,288 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:20:25,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:25,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:25,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:25,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:25,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:25,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:25,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:25,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:25,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:25,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:25,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:25,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:25,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:25,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:25,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:25,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:27,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:27,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:27,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:27,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:27,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:27,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:27,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:27,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:27,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:27,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:27,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:27,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:27,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:27,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:27,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:27,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:28,290 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:20:30,291 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:20:32,291 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:20:34,292 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:20:37,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:20:37,277 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:20:38,294 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:20:40,295 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:20:42,296 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:20:44,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:44,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:44,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:44,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:44,297 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:20:44,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:44,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:44,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:44,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:44,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:44,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:44,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:44,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:44,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 15:20:44,723 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 15:20:44,941 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 15:20:45,018 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 15:20:45,297 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 15:20:46,298 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:20:48,298 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:20:50,299 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:20:52,300 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:20:52,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:20:52,506 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:20:54,301 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:20:54,443 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:20:56,302 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:20:58,303 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:00,304 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:02,305 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:04,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:06,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:07,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:21:07,825 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:21:08,307 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:10,308 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:12,309 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:14,310 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:16,311 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:18,312 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:20,313 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:22,314 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:23,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:21:23,060 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:21:24,314 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:24,522 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:21:26,315 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:28,316 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:30,317 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:32,317 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:34,318 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:36,319 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:38,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:21:38,285 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:21:38,320 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:41,321 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:43,322 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:45,322 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:47,323 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:49,324 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:51,325 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:53,326 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:53,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:21:53,560 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:21:54,601 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:21:55,327 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:57,327 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:21:59,328 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:01,329 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:03,329 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:05,330 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:07,331 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:08,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:22:08,826 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:22:09,332 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:11,333 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:13,334 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:15,334 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:17,335 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:19,336 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:21,337 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:23,338 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:24,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:22:24,051 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:22:24,681 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:22:25,338 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:27,339 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:29,340 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:31,341 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:33,341 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:35,342 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:37,343 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:39,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:22:39,277 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:22:39,344 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:41,345 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:43,346 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:45,346 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:47,347 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:49,348 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:51,349 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:53,350 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:54,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:22:54,569 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:22:54,803 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:22:55,350 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:57,351 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:22:59,352 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:01,353 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:03,353 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:06,355 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:08,355 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:09,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:23:09,879 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:23:10,356 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:12,357 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:14,358 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:16,359 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:18,360 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:20,360 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:22,361 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:24,362 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:24,842 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:23:25,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:23:25,107 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:23:26,363 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:28,364 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:30,364 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:32,365 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:34,366 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:36,367 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:38,368 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:40,369 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:40,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:23:40,400 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:23:42,369 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:44,370 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:46,371 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:48,372 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:50,373 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:52,373 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:54,374 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:54,922 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:23:55,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:23:55,677 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:23:56,375 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:23:58,376 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:00,376 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:02,378 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:04,378 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:07,380 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:09,380 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:10,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:24:10,906 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:24:11,381 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:13,382 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:15,383 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:17,383 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:19,384 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:21,385 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:23,386 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:25,002 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:24:25,386 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:26,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:24:26,179 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:24:27,387 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:29,388 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:31,389 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:33,390 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:35,391 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:37,391 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:39,392 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:41,393 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:41,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:24:41,417 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:24:43,394 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:45,395 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:47,395 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:49,396 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:51,397 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:53,398 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:55,082 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:24:55,399 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:56,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:24:56,787 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:24:57,399 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:24:59,400 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:01,401 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:03,402 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:05,403 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:07,403 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:09,404 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:11,405 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:12,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:25:12,091 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:25:13,406 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:15,407 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:17,407 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:19,408 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:21,409 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:23,410 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:25,161 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:25:25,411 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:27,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:25:27,374 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:25:27,412 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:29,412 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:31,413 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:33,414 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:36,415 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:38,416 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:40,417 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:42,418 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:42,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:25:42,630 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:25:44,418 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:46,419 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:48,420 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:50,421 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:52,422 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:54,422 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:55,240 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:25:56,423 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:25:57,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:25:57,855 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:25:58,424 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:00,425 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:02,426 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:04,426 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:06,427 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:08,428 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:10,429 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:12,430 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:13,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:26:13,180 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:26:14,431 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:16,431 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:18,432 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:20,433 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:22,434 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:24,434 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:25,362 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:26:26,435 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:28,436 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:28,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:26:28,535 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:26:30,437 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:32,438 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:34,438 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:36,439 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:38,440 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:40,440 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:42,441 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:43,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:26:43,758 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:26:44,442 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:46,443 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:48,444 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:50,444 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:52,445 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:54,446 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:55,442 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:26:56,447 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:58,448 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:26:59,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:26:59,070 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:27:00,448 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:02,449 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:04,450 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:06,451 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:08,452 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:10,453 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:13,454 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:14,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:27:14,340 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:27:15,455 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:17,455 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:19,456 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:21,457 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:23,458 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:25,459 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:25,522 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:27:27,459 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:29,460 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:29,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:27:29,564 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:27:31,461 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:33,462 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:35,463 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:37,463 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:39,464 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:41,465 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:43,466 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:44,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:27:44,922 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:27:45,466 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:47,467 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:49,468 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:51,469 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:53,470 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:55,470 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:55,602 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:27:57,471 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:27:59,472 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:00,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:28:00,176 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:28:01,473 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:03,474 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:05,474 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:07,475 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:09,476 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:11,477 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:13,478 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:15,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:28:15,419 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:28:15,478 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:17,479 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:19,480 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:21,481 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:23,482 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:25,483 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:25,681 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:28:27,484 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:29,484 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:30,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:28:30,766 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:28:31,485 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:33,486 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:35,487 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:37,488 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:39,489 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:41,490 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:43,490 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:45,491 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:46,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:28:46,007 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:28:47,492 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:50,493 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:52,494 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:54,495 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:55,760 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:28:56,496 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:28:58,497 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:00,497 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:01,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:29:01,248 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:29:02,498 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:04,499 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:06,500 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:08,501 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:10,502 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:12,502 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:14,503 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:16,504 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:16,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:29:16,645 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:29:18,505 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:20,505 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:22,506 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:24,507 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:25,839 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:29:26,508 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:28,508 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:30,509 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:32,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:29:32,003 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:29:32,510 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:34,511 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:36,512 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:38,512 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:40,513 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:42,514 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:44,515 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:46,515 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:47,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:29:47,226 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:29:48,516 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:50,517 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:52,517 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:54,518 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:55,919 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:29:56,519 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:29:58,520 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:00,520 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:02,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:30:02,529 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:30:03,522 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:05,522 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:07,523 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:09,524 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:11,525 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:13,525 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:15,526 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:17,527 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:17,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:30:17,864 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:30:19,528 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:21,528 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:23,529 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:25,530 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:26,000 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:30:27,531 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:29,531 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:31,532 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:33,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:30:33,127 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:30:33,533 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:35,533 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:37,534 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:39,535 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:41,536 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:43,537 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:45,538 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:47,538 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:48,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:30:48,409 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:30:49,539 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:51,540 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:53,540 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:55,541 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:56,080 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:30:57,542 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:30:59,542 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:01,543 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:03,544 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:03,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:31:03,635 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:31:05,545 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:07,545 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:10,546 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:12,547 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:14,548 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:16,549 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:18,550 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:18,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:31:18,983 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:31:20,550 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:22,551 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:24,552 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:26,159 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:31:26,553 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:28,553 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:30,554 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:32,555 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:34,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:31:34,299 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:31:34,556 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:36,557 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:38,557 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:40,558 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:42,559 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:44,560 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:46,560 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:48,561 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:49,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:31:49,610 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:31:50,562 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:52,563 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:54,563 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:56,240 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:31:56,564 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:31:58,565 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:00,565 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:02,566 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:04,567 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:04,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:32:04,896 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:32:06,568 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:08,569 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:10,570 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:12,571 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:14,572 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:16,573 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:18,574 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:20,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:32:20,163 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:32:20,574 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:22,575 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:24,576 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:26,319 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:32:26,577 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:28,578 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:30,578 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:32,579 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:34,580 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:35,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:32:35,389 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:32:36,580 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:38,581 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:40,582 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:42,582 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:44,583 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:46,584 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:48,584 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:50,585 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:50,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:32:50,691 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:32:52,586 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:54,587 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:56,398 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:32:56,587 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:32:58,588 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:00,589 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:02,590 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:04,591 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:06,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:33:06,090 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:33:06,591 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:09,593 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:11,593 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:13,594 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:15,595 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:17,596 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:19,597 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:21,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:33:21,377 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:33:21,598 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:23,598 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:25,599 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:26,473 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:33:27,600 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:29,601 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:31,601 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:33,602 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:35,603 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:36,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:33:36,671 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:33:37,604 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:39,604 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:41,605 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:43,606 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:45,606 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:47,607 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:49,608 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:51,608 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:51,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:33:51,900 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:33:53,609 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:55,610 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:56,552 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:33:57,610 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:33:59,611 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:01,612 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:03,613 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:05,614 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:07,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:34:07,187 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:34:07,615 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:09,616 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:11,617 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:13,618 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:15,619 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:17,620 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:19,621 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:21,622 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:22,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:34:22,498 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:34:23,623 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:25,624 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:26,631 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:34:27,625 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:29,626 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:32,628 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:34,629 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:36,630 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:37,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:34:37,797 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:34:38,631 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:40,632 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:42,633 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:44,634 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:46,635 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:48,636 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:50,637 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:52,638 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:53,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:34:53,022 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:34:54,639 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:56,640 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:34:56,709 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:34:58,641 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:00,642 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:02,643 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:04,644 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:06,645 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:08,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:35:08,289 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:35:08,646 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:10,648 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:12,648 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:14,649 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:16,650 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:18,651 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:20,652 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:22,653 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:23,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:35:23,589 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:35:24,655 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:26,656 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:26,789 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:35:28,657 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:30,658 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:32,659 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:34,660 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:36,661 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:38,662 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:38,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:35:38,813 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:35:41,663 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:43,664 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:45,665 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:47,666 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:49,667 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:51,668 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:53,669 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:54,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:35:54,094 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:35:55,670 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:56,869 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:35:57,672 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:35:59,673 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:01,674 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:03,675 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:05,676 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:07,677 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:09,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:36:09,322 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:36:09,678 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:11,679 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:13,680 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:15,681 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:17,682 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:19,683 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:21,684 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:23,685 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:24,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:36:24,674 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:36:25,686 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:26,948 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:36:27,687 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:29,688 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:31,689 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:33,690 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:35,691 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:37,692 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:39,693 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:39,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:36:39,901 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:36:41,694 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:43,695 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:45,696 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:47,697 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:49,698 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:51,699 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:54,700 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:55,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:36:55,261 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:36:56,701 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:36:57,027 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:36:58,702 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:00,703 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:02,705 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:04,706 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:06,707 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:08,708 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:10,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:37:10,615 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:37:10,709 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:12,710 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:14,711 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:16,711 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:18,712 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:20,713 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:22,714 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:24,715 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:25,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:37:25,880 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:37:26,716 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:27,105 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:37:28,717 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:30,717 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:32,718 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:34,719 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:36,720 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:38,721 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:40,722 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:41,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:37:41,118 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:37:42,723 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:44,724 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:46,725 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:48,726 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:50,726 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:52,727 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:54,728 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:56,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:37:56,344 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:37:56,729 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:37:57,186 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:37:59,730 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:01,731 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:03,732 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:05,733 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:07,734 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:09,735 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:11,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:38:11,693 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:38:11,736 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:13,736 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:15,737 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:17,738 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:19,739 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:21,740 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:23,741 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:25,741 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:26,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:38:26,986 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:38:27,265 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:38:27,742 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:29,743 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:31,744 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:33,745 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:35,746 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:37,747 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:39,747 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:41,748 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:42,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:38:42,286 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:38:43,749 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:45,750 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:47,751 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:49,752 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:51,753 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:53,753 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:55,754 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:57,343 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:38:57,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:38:57,515 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:38:57,755 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:38:59,756 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:01,757 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:03,758 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:05,759 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:07,759 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:09,760 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:11,761 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:12,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:39:12,882 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:39:13,762 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:15,763 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:17,764 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:19,765 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:21,765 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:23,766 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:26,767 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:27,423 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:39:28,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:39:28,141 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:39:28,768 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:30,769 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:32,770 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:34,771 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:36,772 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:38,773 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:40,773 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:42,774 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:43,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:39:43,460 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:39:44,775 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:46,776 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:48,777 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:50,778 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:52,779 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:54,780 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:56,781 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:39:57,505 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:39:58,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:39:58,749 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:39:58,782 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:00,783 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:02,784 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:04,785 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:06,786 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:08,787 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:10,788 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:12,789 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:13,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:40:13,976 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:40:14,790 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:16,791 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:18,792 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:20,793 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:22,794 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:24,795 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:26,796 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:27,585 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:40:28,797 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:29,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:40:29,210 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:40:30,798 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:32,799 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:34,800 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:36,801 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:38,802 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:41,804 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:43,805 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:44,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:40:44,436 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:40:45,806 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:47,807 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:49,808 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:51,809 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:53,810 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:55,811 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:57,665 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:40:57,812 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:59,813 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:40:59,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:40:59,886 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:41:01,814 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:03,815 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:05,816 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:07,817 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:09,818 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:11,820 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:13,821 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:15,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:41:15,111 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:41:15,822 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:17,823 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:19,824 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:21,825 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:23,826 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:25,827 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:27,745 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:41:27,828 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:29,829 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:30,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:41:30,344 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:41:31,830 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:33,831 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:35,832 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:37,833 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:39,834 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:41,835 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:43,836 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:45,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:41:45,629 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:41:45,837 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:47,838 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:49,839 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:51,840 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:53,841 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:55,842 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:57,824 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:41:57,843 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:41:59,843 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:00,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:42:00,857 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:42:01,844 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:03,845 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:05,846 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:07,847 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:09,848 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:11,849 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:13,850 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:15,851 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:16,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:42:16,088 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:42:17,852 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:19,853 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:21,854 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:23,855 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:26,857 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:27,903 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:42:28,858 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:30,859 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:31,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:42:31,425 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:42:32,860 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:34,861 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:36,862 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:38,863 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:40,864 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:42,865 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:44,866 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:46,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:42:46,776 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:42:46,866 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:48,868 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:50,869 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:52,870 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:54,871 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:56,872 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:42:57,983 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:42:58,872 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:00,873 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:02,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:43:02,029 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:43:02,874 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:04,875 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:06,876 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:08,877 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:10,878 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:12,879 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:14,880 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:17,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:43:17,416 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:43:17,882 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:19,883 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:21,884 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:23,885 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:25,886 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:27,887 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:28,062 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:43:29,888 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:31,889 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:32,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:43:32,642 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:43:33,890 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:35,891 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:37,892 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:39,893 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:41,894 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:43,895 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:45,896 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:47,897 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:48,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:43:48,029 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:43:49,898 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:51,899 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:53,900 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:55,901 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:57,903 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:43:58,139 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:43:59,904 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:01,905 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:03,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:44:03,278 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:44:03,906 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:05,907 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:07,908 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:09,909 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:11,910 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:13,911 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:15,912 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:17,913 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:18,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:44:18,535 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:44:19,914 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:21,915 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:23,916 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:25,917 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:27,917 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:28,220 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:44:30,919 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:32,920 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:33,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:44:33,783 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:44:34,921 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:36,922 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:38,923 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:40,924 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:42,925 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:44,926 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:46,927 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:48,928 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:49,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:44:49,005 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:44:50,929 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:52,930 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:54,931 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:56,932 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:44:58,297 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:44:58,933 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:00,934 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:02,935 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:04,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:45:04,340 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:45:04,936 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:06,937 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:08,938 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:10,939 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:12,940 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:14,941 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:16,942 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:18,943 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:19,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:45:19,566 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:45:20,944 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:22,945 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:24,946 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:26,947 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:28,376 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:45:28,948 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:30,949 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:32,950 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:34,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:45:34,812 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:45:34,951 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:36,952 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:38,953 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:40,954 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:42,955 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:44,956 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:46,957 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:48,958 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:50,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:45:50,104 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:45:50,959 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:52,960 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:54,961 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:57,962 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:45:58,455 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:45:59,963 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:01,964 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:03,965 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:05,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:46:05,436 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:46:05,967 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:07,968 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:09,969 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:11,970 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:13,971 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:15,972 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:17,973 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:19,974 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:20,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:46:20,735 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:46:21,975 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:23,976 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:25,977 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:27,978 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:28,533 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:46:29,979 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:31,980 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:33,981 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:35,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:46:35,961 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:46:35,982 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:37,983 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:39,984 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:41,985 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:43,986 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:45,987 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:47,988 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:49,989 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:51,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:46:51,186 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:46:51,990 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:53,991 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:55,993 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:57,994 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:46:58,611 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:46:59,994 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:01,995 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:03,996 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:05,997 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:06,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:47:06,491 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:47:07,998 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:09,999 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:12,000 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:15,001 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:17,003 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:19,004 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:21,005 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:21,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:47:21,715 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:47:23,005 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:25,006 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:27,008 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:28,689 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:47:29,008 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:31,010 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:33,011 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:35,012 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:36,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:47:36,940 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:47:37,013 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:39,014 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:41,015 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:43,016 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:45,017 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:47,018 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:49,019 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:51,020 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:52,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:47:52,167 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:47:53,021 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:55,022 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:57,023 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:47:58,766 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:47:59,024 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:01,025 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:03,026 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:05,027 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:07,028 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:07,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:48:07,461 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:48:09,029 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:11,030 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:13,031 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:15,032 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:17,033 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:19,034 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:21,035 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:22,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:48:22,691 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:48:23,036 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:25,037 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:27,038 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:28,845 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:48:29,039 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:31,040 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:33,041 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:35,042 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:37,043 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:37,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:48:37,986 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:48:39,044 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:41,045 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:43,046 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:45,047 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:47,048 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:49,049 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:51,050 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:53,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:48:53,217 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:48:54,052 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:56,053 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:58,054 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:48:58,924 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:49:00,055 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:02,056 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:04,057 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:06,058 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:08,059 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:08,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:49:08,444 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:49:10,060 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:12,061 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:14,062 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:16,062 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:18,063 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:20,064 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:22,065 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:23,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:49:23,671 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:49:24,065 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:26,066 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:28,067 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:29,003 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:49:30,068 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:32,069 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:34,070 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:36,070 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:38,071 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:39,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:49:39,019 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:49:40,072 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:42,073 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:44,073 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:46,074 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:48,075 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:50,076 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:52,077 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:54,078 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:54,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:49:54,245 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:49:56,078 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:58,079 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:49:59,081 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:50:00,080 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:02,081 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:04,082 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:06,083 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:08,083 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:09,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:50:09,496 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:50:10,084 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:12,085 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:14,086 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:16,087 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:18,087 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:21,089 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:23,090 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:24,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:50:24,733 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:50:25,090 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:27,091 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:29,092 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:29,161 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:50:31,093 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:33,094 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:35,095 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:37,095 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:39,096 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:40,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:50:40,016 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:50:41,097 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:43,098 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:45,098 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:47,099 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:49,100 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:51,101 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:53,102 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:55,102 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:55,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:50:55,248 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:50:57,103 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:59,104 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:50:59,239 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:51:01,105 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:03,106 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:05,106 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:07,107 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:09,108 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:10,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:51:10,556 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:51:11,109 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:13,110 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:15,110 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:17,111 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:19,112 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:21,113 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:23,114 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:25,114 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:25,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:51:25,906 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:51:27,115 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:29,116 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:29,322 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:51:31,117 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:34,118 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:36,119 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:38,119 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:40,120 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:41,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:51:41,135 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:51:42,121 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:44,122 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:46,123 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:48,124 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:50,124 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:52,125 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:54,126 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:56,127 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:56,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:51:56,503 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:51:58,127 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:51:59,400 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:52:00,128 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:02,129 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:04,130 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:06,131 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:08,131 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:10,132 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:11,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:52:11,762 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:52:12,133 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:14,134 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:16,134 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:18,135 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:20,136 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:22,136 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:24,137 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:26,138 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:26,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:52:26,989 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:52:28,139 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:29,479 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:52:30,140 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:32,140 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:34,141 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:36,142 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:38,143 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:40,143 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:42,144 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:42,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:52:42,216 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:52:44,145 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:46,146 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:49,147 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:51,147 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:53,149 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:55,149 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:57,150 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:57,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:52:57,489 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:52:59,151 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:52:59,557 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:53:01,152 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:03,153 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:05,153 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:07,154 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:09,155 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:11,156 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:12,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:53:12,719 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:53:13,157 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:15,157 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:17,158 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:19,159 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:21,160 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:23,160 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:25,161 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:27,162 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:28,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:53:28,031 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:53:29,163 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:29,636 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:53:31,164 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:33,164 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:35,165 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:37,166 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:39,167 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:41,168 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:43,168 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:43,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:53:43,270 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:53:45,169 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:47,170 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:49,171 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:51,172 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:53,172 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:55,173 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:57,174 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:58,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:53:58,595 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:53:59,175 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:53:59,715 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:54:02,176 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:04,177 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:06,177 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:08,178 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:10,179 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:12,180 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:13,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:54:13,817 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:54:14,181 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:16,181 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:18,182 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:20,183 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:22,184 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:24,185 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:26,186 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:28,186 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:29,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:54:29,091 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:54:29,793 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:54:30,187 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:32,188 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:34,189 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:36,189 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:38,190 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:40,191 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:42,192 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:44,193 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:44,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:54:44,440 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:54:46,193 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:48,194 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:50,195 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:52,196 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:54,196 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:56,197 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:58,198 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:54:59,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:54:59,664 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:54:59,891 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:55:00,198 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:02,199 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:04,200 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:06,201 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:08,201 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:10,202 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:12,203 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:14,204 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:14,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:55:14,893 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:55:16,205 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:18,206 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:20,206 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:22,207 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:25,209 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:27,209 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:29,210 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:29,950 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:55:30,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:55:30,239 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:55:31,211 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:33,212 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:35,213 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:37,214 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:39,215 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:41,216 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:43,217 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:45,218 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:45,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:55:45,501 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:55:47,219 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:49,220 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:51,221 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:53,222 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:55,222 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:57,223 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:55:59,224 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:00,028 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:56:00,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:56:00,725 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:56:01,225 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:03,226 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:05,227 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:07,228 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:09,229 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:11,229 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:13,230 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:15,231 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:16,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:56:16,002 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:56:17,232 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:19,233 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:21,234 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:23,235 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:25,236 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:27,236 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:29,237 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:30,107 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:56:31,238 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:31,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:56:31,273 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:56:33,239 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:35,239 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:37,240 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:39,241 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:41,242 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:43,242 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:45,243 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:46,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:56:46,534 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:56:47,244 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:49,245 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:51,246 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:53,247 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:55,247 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:57,248 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:56:59,249 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:00,185 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:57:01,250 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:01,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:57:01,872 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:57:03,250 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:06,252 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:08,252 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:10,253 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:12,254 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:14,255 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:16,256 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:17,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:57:17,185 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:57:18,257 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:20,257 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:22,258 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:24,259 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:26,260 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:28,261 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:30,262 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:30,264 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:57:32,262 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:32,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:57:32,412 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:57:34,263 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:36,264 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:38,265 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:40,266 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:42,266 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:44,267 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:46,268 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:47,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:57:47,693 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:57:48,269 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:50,270 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:52,271 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:54,272 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:56,272 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:57:58,273 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:00,274 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:00,344 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:58:02,275 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:03,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:58:03,037 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:58:04,275 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:06,276 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:08,277 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:10,278 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:12,279 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:14,280 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:16,280 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:18,281 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:18,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:58:18,321 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:58:20,282 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:22,283 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:24,283 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:26,284 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:28,285 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:30,423 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:58:31,286 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:33,287 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:33,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:58:33,580 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:58:35,288 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:37,289 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:39,289 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:41,290 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:43,291 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:45,292 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:47,293 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:48,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:58:48,805 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:58:49,293 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:51,294 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:53,295 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:55,296 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:57,297 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:58:59,297 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:00,501 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:59:01,298 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:03,299 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:04,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:59:04,130 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:59:05,300 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:07,301 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:09,302 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:11,302 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:13,303 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:15,304 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:17,305 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:19,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:19,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:59:19,404 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:59:21,307 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:23,308 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:25,308 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:27,309 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:29,310 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:30,579 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 15:59:31,311 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:33,312 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:34,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:59:34,628 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:59:35,313 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:37,314 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:39,315 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:41,316 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:43,317 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:45,318 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:47,319 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:49,320 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:49,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 15:59:49,997 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 15:59:51,321 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:53,322 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:55,323 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 15:59:58,324 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:00,326 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:00,657 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:00:02,327 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:04,328 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:05,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:00:05,279 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:00:06,329 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:08,330 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:10,331 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:12,332 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:14,333 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:16,334 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:18,335 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:20,336 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:20,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:00:20,667 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:00:22,337 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:24,338 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:26,339 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:28,340 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:30,341 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:30,736 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:00:32,342 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:34,343 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:35,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:00:35,962 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:00:36,344 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:38,345 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:40,346 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:42,348 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:44,348 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:46,349 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:48,351 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:50,352 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:51,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:00:51,187 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:00:52,353 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:54,354 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:56,355 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:00:58,356 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:00,357 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:00,814 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:01:02,358 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:04,359 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:06,360 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:06,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:01:06,465 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:01:08,361 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:10,362 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:12,362 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:15,364 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:17,365 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:19,366 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:21,367 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:21,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:01:21,748 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:01:23,368 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:25,369 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:27,370 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:29,371 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:30,892 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:01:31,372 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:33,373 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:35,374 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:37,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:01:37,068 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:01:37,375 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:39,376 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:41,377 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:43,378 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:45,379 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:47,381 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:49,381 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:51,382 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:52,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:01:52,350 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:01:53,383 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:55,384 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:57,385 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:01:59,386 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:00,970 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:02:02,388 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:04,389 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:06,390 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:07,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:02:07,577 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:02:08,391 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:10,392 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:12,393 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:14,394 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:16,395 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:18,396 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:20,397 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:22,398 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:22,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:02:22,802 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:02:24,399 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:26,400 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:28,400 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:30,401 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:31,053 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:02:32,402 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:34,403 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:36,404 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:38,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:02:38,079 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:02:38,405 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:40,406 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:42,407 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:44,408 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:46,409 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:48,410 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:50,411 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:52,412 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:53,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:02:53,404 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:02:54,413 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:56,414 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:02:58,415 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:00,416 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:01,130 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:03:02,417 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:04,418 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:07,420 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:08,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:03:08,628 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:03:09,421 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:11,422 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:13,423 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:15,424 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:17,425 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:19,426 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:21,427 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:23,428 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:23,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:03:23,860 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:03:25,429 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:27,430 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:29,431 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:31,204 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:03:31,432 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:33,433 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:35,434 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:37,435 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:39,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:03:39,191 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:03:39,436 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:41,437 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:43,438 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:45,439 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:47,440 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:49,441 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:51,442 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:53,443 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:54,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:03:54,414 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:03:55,444 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:57,445 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:03:59,446 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:01,279 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:04:01,447 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:03,448 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:05,449 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:07,449 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:09,450 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:09,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:04:09,718 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:04:11,451 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:13,452 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:15,453 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:17,454 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:19,455 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:21,456 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:23,457 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:24,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:04:24,940 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:04:25,458 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:28,460 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:30,461 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:31,354 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:04:32,462 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:34,462 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:36,463 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:38,465 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:40,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:04:40,166 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:04:40,466 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:42,467 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:44,468 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:46,469 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:48,470 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:50,471 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:52,472 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:54,473 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:55,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:04:55,393 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:04:56,474 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:04:58,475 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:00,476 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:01,429 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:05:02,477 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:04,478 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:06,478 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:08,480 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:10,481 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:10,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:05:10,653 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:05:12,482 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:14,482 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:16,483 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:18,484 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:20,485 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:22,486 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:24,487 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:25,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:05:25,919 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:05:26,488 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:28,489 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:30,490 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:31,506 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:05:32,491 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:34,492 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:36,492 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:38,493 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:40,494 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:41,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:05:41,204 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:05:42,495 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:44,496 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:46,497 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:48,498 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:50,499 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:52,500 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:55,502 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:56,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:05:56,443 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:05:57,503 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:05:59,504 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:01,505 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:01,583 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:06:03,506 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:05,507 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:07,508 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:09,509 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:11,510 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:11,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:06:11,733 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:06:13,511 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:15,512 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:17,513 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:19,514 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:21,515 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:23,516 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:25,517 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:26,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:06:26,959 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:06:27,518 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:29,519 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:31,520 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:31,662 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:06:33,522 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:35,523 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:37,524 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:39,525 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:41,526 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:42,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:06:42,263 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:06:43,527 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:45,528 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:47,529 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:49,530 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:51,531 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:53,532 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:55,533 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:57,534 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:06:57,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:06:57,574 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:06:59,535 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:01,536 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:01,741 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:07:03,537 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:05,538 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:07,539 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:09,540 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:12,541 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:12,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:07:12,801 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:07:14,542 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:16,543 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:18,544 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:20,545 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:22,546 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:24,547 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:26,548 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:28,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:07:28,027 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:07:28,549 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:30,550 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:31,818 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:07:32,551 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:34,551 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:36,552 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:38,553 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:40,554 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:42,555 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:43,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:07:43,314 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:07:44,557 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:46,558 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:48,559 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:50,560 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:52,560 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:54,561 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:56,562 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:58,563 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:07:58,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:07:58,589 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:08:00,564 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:01,900 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:08:02,565 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:04,566 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:06,567 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:08,568 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:10,569 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:12,570 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:13,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:08:13,813 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:08:14,571 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:16,571 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:18,572 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:20,573 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:22,574 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:24,575 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:26,576 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:28,577 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:29,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:08:29,148 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:08:30,578 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:31,978 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:08:32,579 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:34,580 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:36,581 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:38,582 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:40,583 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:42,584 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:44,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:08:44,427 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:08:44,584 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:46,585 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:48,586 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:50,587 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:53,588 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:55,589 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:57,590 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:59,591 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:08:59,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:08:59,651 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:09:01,592 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:02,057 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:09:03,593 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:05,594 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:07,595 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:09,596 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:11,597 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:13,598 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:15,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:09:15,015 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:09:15,599 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:17,600 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:19,601 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:21,602 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:23,603 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:25,604 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:27,604 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:29,605 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:30,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:09:30,314 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:09:31,606 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:32,136 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:09:33,607 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:35,608 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:37,609 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:39,610 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:42,611 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:44,612 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:45,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:09:45,541 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:09:46,612 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:48,613 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:50,614 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:52,614 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:54,615 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:56,616 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:09:58,616 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:00,617 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:00,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:10:00,778 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:10:02,215 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:10:02,618 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:04,619 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:06,619 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:08,620 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:10,621 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:12,621 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:14,622 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:16,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:10:16,003 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:10:16,623 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:18,623 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:20,624 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:22,625 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:24,625 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:26,626 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:28,627 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:30,627 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:31,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:10:31,278 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:10:32,294 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:10:32,628 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:34,628 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:36,629 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:39,630 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:41,631 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:43,631 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:45,632 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:46,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:10:46,615 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:10:47,633 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:49,633 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:51,634 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:53,635 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:55,635 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:57,636 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:10:59,637 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:01,637 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:01,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:11:01,912 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:11:02,374 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:11:03,638 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:05,639 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:07,639 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:09,640 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:11,641 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:13,642 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:15,643 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:17,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:11:17,144 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:11:17,644 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:19,645 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:21,645 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:23,646 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:25,647 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:27,648 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:29,648 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:31,649 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:32,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:11:32,444 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:11:32,692 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:11:33,650 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:35,651 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:37,652 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:39,652 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:41,653 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:43,654 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:45,654 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:47,655 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:47,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:11:47,694 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:11:49,656 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:51,657 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:54,658 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:56,658 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:11:58,659 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:00,660 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:02,531 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:12:02,661 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:02,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:12:02,918 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:12:04,661 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:06,662 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:08,663 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:10,664 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:12,665 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:14,665 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:16,666 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:18,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:12:18,176 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:12:18,667 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:20,668 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:22,669 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:24,670 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:26,671 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:28,672 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:30,673 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:32,610 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:12:32,674 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:33,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:12:33,485 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:12:34,675 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:36,676 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:38,677 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:40,677 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:42,678 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:44,679 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:46,680 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:48,680 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:48,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:12:48,716 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:12:50,681 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:52,682 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:54,683 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:56,684 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:12:58,684 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:00,685 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:02,686 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:02,690 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:13:04,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:13:04,095 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:13:04,687 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:06,688 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:08,688 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:10,689 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:12,690 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:14,691 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:16,691 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:18,692 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:19,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:13:19,410 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:13:20,693 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:22,694 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:24,695 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:26,696 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:28,697 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:30,698 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:32,698 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:32,770 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:13:34,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:13:34,672 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:13:34,699 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:36,700 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:39,702 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:41,702 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:43,703 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:45,704 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:47,705 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:49,706 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:49,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:13:49,895 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:13:51,707 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:53,708 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:55,709 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:57,710 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:13:59,711 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:01,712 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:02,852 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:14:03,712 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:05,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:14:05,181 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:14:05,713 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:07,714 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:09,715 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:11,716 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:13,717 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:15,717 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:17,718 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:19,719 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:20,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:14:20,455 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:14:21,720 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:23,721 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:25,722 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:27,723 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:29,723 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:31,724 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:32,931 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:14:33,725 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:35,726 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:35,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:14:35,728 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:14:37,727 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:39,727 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:42,729 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:44,730 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:46,730 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:48,731 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:50,732 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:50,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:14:50,952 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:14:52,733 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:54,734 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:56,735 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:14:58,736 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:00,737 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:02,737 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:03,011 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:15:04,738 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:06,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:15:06,272 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:15:06,739 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:08,740 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:10,741 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:12,742 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:14,742 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:16,743 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:18,744 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:20,745 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:21,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:15:21,610 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:15:22,746 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:24,747 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:26,748 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:28,749 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:30,749 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:32,750 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:33,091 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:15:34,751 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:36,752 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:36,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:15:36,835 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:15:38,753 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:40,753 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:42,754 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:44,755 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:46,756 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:48,757 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:50,758 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:52,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:15:52,146 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:15:52,758 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:55,760 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:57,761 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:15:59,762 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:01,763 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:03,170 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:16:03,764 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:05,765 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:07,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:16:07,377 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:16:07,766 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:09,767 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:11,768 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:13,768 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:15,769 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:17,770 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:19,771 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:21,772 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:22,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:16:22,605 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:16:23,773 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:25,774 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:27,775 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:29,775 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:31,776 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:33,250 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:16:33,777 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:35,778 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:37,779 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:37,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:16:37,928 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:16:39,780 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:41,781 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:43,782 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:45,783 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:47,783 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:49,784 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:51,785 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:53,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:16:53,153 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:16:53,786 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:55,787 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:57,788 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:16:59,789 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:01,790 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:03,330 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:17:03,791 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:05,792 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:07,792 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:08,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:17:08,547 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:17:09,793 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:12,794 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:14,795 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:16,796 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:18,797 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:20,798 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:22,799 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:23,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:17:23,786 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:17:24,800 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:26,801 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:28,802 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:30,803 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:32,804 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:33,408 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:17:34,805 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:36,806 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:38,807 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:39,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:17:39,100 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:17:40,807 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:42,808 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:44,809 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:46,810 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:48,811 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:50,812 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:52,813 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:54,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:17:54,388 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:17:54,814 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:56,814 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:17:58,815 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:00,816 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:02,817 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:03,487 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:18:04,818 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:06,818 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:08,819 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:09,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:18:09,721 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:18:10,820 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:12,821 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:14,822 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:16,822 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:18,823 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:20,824 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:22,825 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:24,825 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:25,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:18:25,016 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:18:26,826 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:28,827 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:30,828 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:32,828 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:33,567 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:18:34,829 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:36,830 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:38,831 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:40,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:18:40,239 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:18:40,831 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:42,832 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:45,833 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:47,834 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:49,835 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:51,836 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:53,837 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:55,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:18:55,568 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:18:55,837 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:57,838 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:18:59,839 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:01,840 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:03,645 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:19:03,841 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:05,842 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:07,842 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:09,843 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:10,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:19:10,819 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:19:11,844 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:13,845 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:15,846 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:17,847 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:19,848 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:21,849 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:23,850 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:25,851 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:26,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:19:26,150 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:19:27,852 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:29,853 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:31,853 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:33,723 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:19:33,854 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:35,855 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:37,856 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:39,857 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:41,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:19:41,373 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:19:41,858 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:43,859 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:45,860 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:47,861 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:49,862 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:51,863 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:53,864 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:55,865 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:56,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:19:56,761 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:19:57,866 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:19:59,867 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:01,868 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:03,802 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:20:03,869 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:05,870 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:07,871 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:09,872 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:11,873 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:11,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:20:11,988 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:20:13,874 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:15,875 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:17,876 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:19,877 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:21,878 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:23,879 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:25,880 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:27,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:20:27,301 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:20:27,881 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:30,882 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:32,883 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:33,881 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:20:34,884 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:36,885 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:38,886 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:40,886 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:42,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:20:42,571 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:20:42,888 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:44,889 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:46,889 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:48,891 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:50,892 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:52,892 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:54,893 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:56,894 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:20:57,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:20:57,893 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:20:58,897 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:00,898 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:02,899 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:03,964 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:21:04,900 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:06,901 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:08,902 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:10,903 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:12,904 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:13,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:21:13,119 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:21:14,905 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:16,906 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:18,907 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:20,908 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:22,908 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:24,909 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:26,910 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:28,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:21:28,370 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:21:29,912 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:31,913 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:33,914 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:34,040 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:21:35,915 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:37,916 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:39,916 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:41,917 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:43,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:21:43,603 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:21:43,918 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:45,919 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:47,920 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:49,920 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:51,921 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:53,922 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:55,923 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:57,924 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:21:58,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:21:58,829 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:21:59,925 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:01,926 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:03,926 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:04,118 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:22:05,927 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:07,928 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:09,929 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:11,930 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:13,930 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:14,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:22:14,057 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:22:15,931 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:17,932 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:19,933 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:21,934 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:23,934 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:25,935 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:27,936 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:29,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:22:29,397 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:22:29,937 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:31,937 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:33,938 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:34,199 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:22:35,938 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:38,939 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:40,940 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:42,941 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:44,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:22:44,704 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:22:44,941 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:46,942 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:48,943 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:50,943 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:52,944 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:54,945 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:56,945 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:58,946 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:22:59,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:22:59,926 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:23:00,947 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:02,947 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:04,278 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:23:04,948 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:06,949 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:08,949 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:10,950 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:12,951 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:14,951 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:15,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:23:15,229 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:23:16,952 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:18,952 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:20,953 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:22,954 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:24,954 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:26,955 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:28,956 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:30,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:23:30,478 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:23:30,956 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:32,957 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:34,359 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:23:34,958 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:36,958 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:38,959 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:40,960 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:42,960 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:44,961 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:45,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:23:45,702 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:23:46,961 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:48,962 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:50,963 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:52,964 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:54,964 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:56,965 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:23:58,965 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:00,966 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:01,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:24:01,004 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:24:02,967 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:04,438 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:24:04,968 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:07,968 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:09,969 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:11,970 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:13,970 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:15,971 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:16,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:24:16,228 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:24:17,972 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:19,972 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:21,973 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:23,974 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:25,974 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:27,975 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:29,976 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:31,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:24:31,588 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:24:31,976 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:33,977 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:34,517 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:24:35,978 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:37,978 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:39,979 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:41,980 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:43,980 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:45,981 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:46,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:24:46,817 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:24:47,981 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:49,982 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:51,983 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:53,983 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:55,984 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:57,985 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:24:59,985 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:01,986 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:02,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:25:02,130 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:25:03,987 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:04,596 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:25:05,987 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:07,988 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:10,989 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:12,990 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:14,990 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:16,991 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:17,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:25:17,443 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:25:18,992 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:20,992 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:22,993 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:24,993 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:26,994 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:28,995 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:30,996 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:32,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:25:32,684 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:25:32,996 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:34,677 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:25:34,997 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:36,997 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:38,998 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:40,999 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:42,999 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:45,000 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:47,001 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:47,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:25:47,977 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:25:49,002 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:51,002 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:53,003 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:55,004 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:57,004 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:25:59,005 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:01,006 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:03,006 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:03,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:26:03,242 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:26:04,755 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:26:05,007 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:07,007 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:09,008 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:11,009 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:13,010 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:15,011 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:17,012 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:18,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:26:18,539 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:26:19,013 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:21,014 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:23,015 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:25,016 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:27,017 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:29,018 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:31,019 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:33,020 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:33,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:26:33,777 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:26:34,833 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:26:35,021 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:37,022 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:39,023 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:41,024 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:43,024 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:45,025 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:47,026 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:49,028 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:49,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:26:49,056 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:26:51,028 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:53,029 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:55,030 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:57,032 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:26:59,033 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:01,034 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:03,035 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:04,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:27:04,443 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:27:04,912 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:27:05,036 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:08,037 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:10,038 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:12,039 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:14,040 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:16,041 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:18,042 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:19,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:27:19,709 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:27:20,043 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:22,045 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:24,046 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:26,047 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:28,048 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:30,049 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:32,050 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:34,051 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:34,992 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:27:35,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:27:35,006 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:27:36,052 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:38,053 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:40,054 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:42,055 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:44,056 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:46,057 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:48,058 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:50,059 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:50,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:27:50,378 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:27:52,060 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:54,061 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:56,062 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:27:58,063 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:00,064 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:02,065 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:04,066 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:05,072 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:28:05,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:28:05,604 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:28:06,067 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:08,068 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:10,069 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:12,070 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:14,071 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:16,072 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:18,073 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:21,075 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:21,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:28:21,906 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:28:23,076 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:25,077 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:27,078 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:29,079 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:31,080 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:33,081 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:35,082 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:35,150 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:28:37,083 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:37,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:28:37,177 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:28:39,084 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:41,085 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:43,086 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:45,087 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:47,088 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:49,089 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:51,090 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:52,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:28:52,410 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:28:53,091 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:55,092 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:57,093 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:28:59,094 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:01,095 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:03,096 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:05,097 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:05,230 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:29:07,098 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:07,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:29:07,696 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:29:09,099 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:11,100 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:13,101 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:15,102 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:17,104 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:19,105 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:21,106 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:22,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:29:22,973 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:29:23,107 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:25,108 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:27,109 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:29,110 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:31,111 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:33,112 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:35,113 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:35,312 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:29:37,114 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:38,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:29:38,301 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:29:39,115 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:41,116 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:43,117 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:45,118 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:47,120 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:50,122 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:52,123 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:53,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:29:53,534 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:29:54,124 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:56,125 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:29:58,126 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:00,127 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:02,128 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:04,129 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:05,394 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:30:06,130 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:08,131 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:08,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:30:08,803 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:30:10,132 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:12,133 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:14,134 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:16,135 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:18,136 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:20,137 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:22,138 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:24,139 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:24,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:30:24,179 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:30:26,140 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:28,141 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:30,142 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:32,143 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:34,144 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:35,474 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:30:36,145 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:38,146 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:39,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:30:39,408 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:30:40,147 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:42,148 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:44,149 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:46,150 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:48,151 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:50,151 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:53,153 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:54,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:30:54,645 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:30:55,153 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:57,154 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:30:59,155 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:01,156 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:03,157 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:05,157 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:05,554 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:31:07,158 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:09,159 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:09,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:31:09,934 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:31:11,160 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:13,160 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:15,161 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:17,162 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:19,163 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:21,164 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:23,165 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:25,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:31:25,154 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:31:25,166 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:27,167 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:29,167 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:31,168 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:33,169 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:35,169 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:35,634 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:31:37,170 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:39,171 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:40,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:31:40,427 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:31:41,172 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:43,173 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:45,173 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:47,174 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:49,175 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:31:55,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:31:55,651 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:32:05,714 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:32:10,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:32:10,887 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:32:26,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:32:26,140 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:32:35,792 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:32:41,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:32:41,412 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:32:56,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:32:56,692 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:33:03,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:33:03,796 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 16:33:03,848 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 16:33:03,928 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 16:33:04,203 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 16:33:05,204 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:33:05,936 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:33:06,204 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:33:11,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:33:11,952 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:33:12,207 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:33:27,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:33:27,182 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:33:28,215 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:33:36,023 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:33:42,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:33:42,407 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:33:57,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:33:57,638 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:34:06,107 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:34:12,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:34:12,862 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:34:28,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:34:28,089 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:34:36,192 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:34:43,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:34:43,319 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:34:58,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:34:58,549 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:35:06,280 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:35:13,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:35:13,781 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:35:29,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:35:29,007 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:35:36,365 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:35:44,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:35:44,236 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:35:59,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:35:59,465 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:36:06,450 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:36:14,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:36:14,693 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:36:29,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:36:29,927 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:36:36,534 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:36:45,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:36:45,152 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:37:00,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:37:00,384 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:37:06,620 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:37:15,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:37:15,612 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:37:30,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:37:30,839 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:37:32,337 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:37:34,338 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:37:36,339 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:37:36,703 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:37:38,339 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:37:40,340 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:37:42,341 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:37:44,342 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:37:46,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:37:46,121 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:37:46,346 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:37:48,347 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:37:50,348 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:37:52,348 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:37:54,349 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:37:56,350 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:37:56,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:37:56,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:37:56,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:37:56,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:37:56,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:37:56,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:37:56,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:37:56,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:37:56,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:37:56,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:37:56,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:37:56,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:37:56,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:37:56,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:37:56,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:37:56,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:37:58,351 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:38:00,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:00,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:00,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:00,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:00,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:00,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:00,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:00,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:00,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:00,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:00,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:00,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:00,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:00,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:00,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:00,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:00,351 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:38:01,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:38:01,445 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:38:02,352 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:38:04,353 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:38:04,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:04,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:04,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:04,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:04,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:04,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:04,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:04,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:04,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:04,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:04,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:04,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:04,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:04,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:04,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:04,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:06,354 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:38:06,781 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:38:08,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,354 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:38:08,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:08,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:09,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:10,355 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:38:11,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:11,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,355 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:38:12,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:12,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:13,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:14,356 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:38:15,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:15,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:15,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:15,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:15,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:15,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:15,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:15,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:15,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:15,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:15,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:15,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:15,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:15,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:15,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:15,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:16,357 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:38:17,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:38:17,156 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:38:20,359 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:38:20,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:20,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:20,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:20,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:20,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:20,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:20,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:20,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:20,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:20,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:20,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:20,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:20,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:20,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:20,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:20,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:22,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:22,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:22,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:22,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:22,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:22,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:22,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:22,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:22,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:22,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:22,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:22,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:22,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:22,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:22,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:22,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:24,360 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:38:25,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:25,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:25,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:25,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:25,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:25,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:25,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:25,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:25,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:25,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:25,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:25,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:25,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:25,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:25,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:25,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:26,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:26,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:26,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:26,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:26,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:26,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:26,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:26,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:26,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:26,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:26,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:26,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:26,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:26,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:26,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:26,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:27,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:27,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:27,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:27,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:27,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:27,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:27,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:27,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:27,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:27,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:27,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:27,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:27,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:27,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:27,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:27,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:28,362 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:38:30,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:30,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:32,363 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:38:32,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:38:32,551 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:38:34,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:34,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:34,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:34,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:34,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:34,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:34,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:34,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:34,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:34,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:34,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:34,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:34,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:34,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:34,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:34,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:35,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:36,365 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:38:36,853 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:38:39,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:39,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:39,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:39,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:39,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:39,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:39,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:39,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:39,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:39,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:39,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:39,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:39,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:39,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:39,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:39,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,366 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:38:40,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:40,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:42,367 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:38:43,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:43,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:43,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:43,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:43,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:43,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:43,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:43,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:43,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:43,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:43,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:43,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:43,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:43,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:43,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:43,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:45,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:45,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:45,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:45,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:45,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:45,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:45,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:45,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:45,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:45,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:45,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:45,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:45,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:45,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:45,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:45,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,369 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:38:46,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:46,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:47,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:38:47,779 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:38:48,370 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:38:49,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:49,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:53,372 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:38:53,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:53,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:53,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:53,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:54,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:55,373 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:38:57,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:57,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:57,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:57,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:57,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:57,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:57,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:57,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:57,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:57,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:57,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:57,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:57,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:57,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:57,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:57,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:58,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:58,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:58,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:58,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:58,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:58,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:58,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:58,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:58,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:58,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:58,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:58,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:58,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:58,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:58,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:58,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:38:59,374 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:01,375 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:02,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:02,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:02,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:02,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:02,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:02,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:02,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:02,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:02,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:02,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:02,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:02,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:02,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:02,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:02,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:02,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:03,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:39:03,093 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:39:03,376 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:05,377 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:06,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:06,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:06,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:06,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:06,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:06,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:06,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:06,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:06,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:06,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:06,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:06,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:06,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:06,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:06,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:06,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:06,935 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:39:09,379 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:11,380 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:13,380 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:15,381 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:15,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:15,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:15,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:15,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:16,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:16,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:16,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:16,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:16,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:16,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:16,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:16,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:18,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:39:18,320 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:39:19,383 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:21,384 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:23,385 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:25,385 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:27,386 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:29,387 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:31,388 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:33,389 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:33,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:39:33,552 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:39:35,390 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:37,009 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:39:37,391 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:39,392 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:41,392 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:43,393 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:45,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:45,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:45,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:45,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:45,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:45,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:45,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:45,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:45,394 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:45,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:45,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:45,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:45,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:45,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:45,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:45,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:45,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:47,395 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:48,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:39:48,779 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:39:49,396 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:50,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:50,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:50,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:50,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:50,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:50,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:50,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:50,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:50,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:50,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:50,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:50,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:50,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:50,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:50,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:50,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:51,396 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:52,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:52,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:52,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:52,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:52,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:52,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:52,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:52,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:52,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:52,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:52,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:52,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:52,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:52,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:52,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:52,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:53,397 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:55,398 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:56,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:56,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:56,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:56,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:56,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:56,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:56,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:56,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:56,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:56,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:56,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:56,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:56,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:56,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:56,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:56,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:57,399 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:57,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:57,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:57,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:57,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:57,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:57,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:57,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:57,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:57,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:57,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:57,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:57,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:57,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:57,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:57,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:57,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:58,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:58,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:58,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:58,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:58,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:58,448 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:58,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:58,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:58,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:58,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:58,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:58,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:58,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:58,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:58,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:58,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,399 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:39:59,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:39:59,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:00,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,400 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:40:01,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:01,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:03,401 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:40:04,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:40:04,038 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:40:05,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:05,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:05,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:05,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:05,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:05,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:05,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:05,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:05,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:05,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:05,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:05,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:05,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:05,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:05,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:05,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:07,088 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:40:08,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:08,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:08,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:08,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:08,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:08,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:08,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:08,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:08,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:08,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:08,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:08,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:08,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:08,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:08,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:08,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:09,403 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:40:10,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:10,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:10,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:10,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:10,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:10,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:10,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:10,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:10,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:10,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:10,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:10,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:10,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:10,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:10,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:10,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:13,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:13,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:13,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:13,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:13,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:13,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:13,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:13,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:13,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:13,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:13,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:13,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:13,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:13,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:13,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:13,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:14,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:14,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:14,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:14,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:14,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:14,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:14,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:14,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:14,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:14,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:14,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:14,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:14,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:14,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:14,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:14,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:15,406 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:40:17,406 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:40:19,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:40:19,274 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:40:20,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:20,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:20,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:20,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:20,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:20,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:20,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:20,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:20,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:20,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:20,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:20,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:20,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:20,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:20,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:20,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:21,408 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:40:21,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:21,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:21,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:21,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:21,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:21,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:21,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:21,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:21,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:21,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:21,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:21,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:21,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:21,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:21,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:21,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:23,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:24,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:24,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:24,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:24,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:24,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:24,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:24,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:24,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:24,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:24,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:24,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:24,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:24,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:24,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:24,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:24,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,409 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:40:25,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:25,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:29,411 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:40:31,412 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:40:31,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:31,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:31,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:31,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:31,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:31,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:31,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:31,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:31,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:31,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:31,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:31,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:31,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:31,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:31,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:31,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:32,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:32,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:32,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:32,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:32,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:32,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:32,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:32,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:32,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:32,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:32,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:32,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:32,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:32,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:32,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:32,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:33,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:34,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:40:34,508 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:40:35,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:35,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:35,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:35,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:35,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:35,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:35,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:35,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:35,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:35,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:35,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:35,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:35,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:35,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:35,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:35,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:36,414 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:40:36,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:36,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:36,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:36,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:36,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:36,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:36,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:36,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:36,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:36,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:36,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:36,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:36,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:36,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:36,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:36,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:37,165 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:40:37,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:37,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:37,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:37,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:37,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:37,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:37,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:37,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:37,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:37,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:37,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:37,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:37,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:37,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:37,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:37,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:38,415 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:40:41,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:41,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:41,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:41,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:41,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:41,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:41,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:41,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:41,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:41,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:41,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:41,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:41,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:41,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:41,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:41,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:42,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:42,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:42,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:42,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:42,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:42,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:42,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:42,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:42,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:42,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:42,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:42,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:42,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:42,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:42,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:42,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:42,416 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:40:43,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:43,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:43,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:43,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:43,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:43,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:43,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:43,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:43,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:43,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:43,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:43,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:43,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:43,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:43,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:43,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:44,417 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:40:46,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:46,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:46,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:46,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:46,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:46,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:46,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:46,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:46,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:46,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:46,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:46,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:46,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:46,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:46,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:46,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:47,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:47,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:47,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:47,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:47,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:47,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:47,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:47,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:47,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:47,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:47,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:47,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:47,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:47,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:47,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:47,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,419 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:40:48,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:48,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:49,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:40:49,736 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:40:50,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:50,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:50,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:50,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:50,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:50,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:50,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:50,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:50,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:50,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:50,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:50,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:50,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:50,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:50,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:50,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:50,420 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:40:51,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:51,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:51,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:51,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:51,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:51,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:51,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:51,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:51,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:51,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:51,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:51,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:51,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:51,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:51,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:51,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:52,421 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:40:54,422 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:40:57,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:57,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:57,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:57,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:57,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:57,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:57,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:57,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:57,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:57,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:57,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:57,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:57,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:57,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:57,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:57,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:40:58,423 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:00,424 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:00,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:00,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:00,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:00,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:00,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:00,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:00,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:01,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:01,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:01,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:01,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:01,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:01,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:01,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:01,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:01,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:02,425 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:04,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:04,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:04,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:04,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:04,426 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:04,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:04,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:04,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:04,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:04,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:04,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:04,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:04,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:04,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:04,748 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 16:41:04,967 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 16:41:04,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:41:05,039 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 16:41:05,043 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:41:05,427 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 16:41:06,427 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:07,256 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:41:08,428 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:10,429 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:12,429 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:14,430 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:16,431 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:18,432 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:20,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:41:20,263 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:41:20,433 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:22,434 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:24,435 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:26,435 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:28,436 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:30,437 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:32,438 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:33,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:33,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:33,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:33,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:33,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:33,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:33,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:33,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:33,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:33,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:33,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:33,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:33,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:33,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:33,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:33,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:34,439 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:35,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:41:35,492 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:41:36,440 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:37,335 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:41:38,440 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:39,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:39,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:39,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:39,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:39,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:39,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:39,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:39,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:39,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:39,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:39,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:39,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:39,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:39,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:39,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:39,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:40,441 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:42,442 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:44,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:44,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:44,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:44,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:44,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:44,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:44,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:44,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:44,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:44,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:44,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:44,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:44,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:44,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:44,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:44,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:44,442 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:45,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:45,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:45,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:45,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:45,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:45,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:45,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:45,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:45,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:45,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:45,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:45,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:45,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:45,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:45,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:45,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:46,443 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:46,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:46,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:46,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:46,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:46,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:46,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:46,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:46,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:46,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:46,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:46,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:46,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:46,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:46,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:46,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:46,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:47,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,444 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:48,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:48,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:50,445 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:50,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:41:50,727 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:41:50,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:50,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:50,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:50,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:50,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:50,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:50,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:50,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:51,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:51,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:51,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:51,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:51,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:51,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:51,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:51,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:53,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:53,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:53,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:53,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:53,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:53,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:53,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:53,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:53,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:53,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:53,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:53,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:53,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:53,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:53,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:53,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:55,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:55,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:55,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:55,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:55,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:55,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:55,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:55,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:55,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:55,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:55,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:55,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:55,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:55,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:55,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:55,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:56,447 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:41:59,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:41:59,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:00,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:00,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:00,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:00,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:00,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:00,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:00,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:00,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:00,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:00,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:00,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:00,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:00,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:00,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:00,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:00,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:00,449 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:42:01,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:01,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:04,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:04,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:04,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:04,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:04,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:04,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:04,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:04,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:04,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:04,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:04,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:04,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:04,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:04,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:04,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:04,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:05,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:05,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:05,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:05,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:05,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:05,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:05,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:05,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:05,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:05,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:05,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:05,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:05,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:05,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:05,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:05,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:05,451 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:42:06,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:42:06,063 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:42:07,415 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:42:07,452 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:42:10,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:10,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:10,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:10,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:10,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:10,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:10,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:10,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:10,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:10,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:10,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:10,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:10,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:10,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:10,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:10,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:11,454 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:42:11,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:11,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:11,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:11,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:11,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:11,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:11,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:11,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:11,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:11,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:11,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:11,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:11,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:11,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:11,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:11,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:14,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:15,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:15,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:15,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:15,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:15,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:15,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:15,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:15,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:15,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:15,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:15,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:15,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:15,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:15,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:15,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:15,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:15,455 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:42:16,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:16,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:16,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:16,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:16,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:16,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:16,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:16,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:16,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:16,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:16,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:16,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:16,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:16,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:16,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:16,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:17,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:17,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:17,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:17,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:17,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:17,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:17,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:17,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:17,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:17,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:17,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:17,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:17,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:17,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:17,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:17,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:17,456 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:42:19,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:19,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:21,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:42:21,322 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:42:21,458 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:42:22,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:22,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:22,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:22,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:22,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:22,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:22,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:22,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:22,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:22,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:22,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:22,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:22,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:22,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:22,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:22,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:23,459 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:42:23,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:23,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:23,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:23,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:23,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:23,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:23,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:23,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:23,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:23,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:23,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:23,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:23,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:23,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:23,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:23,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:25,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:26,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:26,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:26,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:26,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:26,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:26,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:26,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:26,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:26,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:26,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:26,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:26,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:26,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:26,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:26,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:26,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:27,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:27,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:27,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:27,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:27,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:27,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:27,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:27,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:27,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:27,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:27,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:27,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:27,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:27,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:27,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:27,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:27,471 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:42:29,472 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:42:31,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:31,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:31,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:31,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:31,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:31,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:31,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:31,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:31,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:31,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:31,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:31,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:31,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:31,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:31,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:31,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:31,473 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:42:33,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:33,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:33,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:33,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:33,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:33,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:33,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:33,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:33,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:33,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:33,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:33,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:33,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:33,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:33,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:33,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:34,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:34,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:34,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:34,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:34,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:34,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:34,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:34,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:34,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:34,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:34,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:34,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:34,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:34,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:34,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:34,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:35,474 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:42:36,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:36,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:36,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:36,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:36,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:36,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:36,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:36,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:36,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:36,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:36,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:36,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:36,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:36,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:36,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:36,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:36,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:42:36,635 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:42:37,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,475 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:42:37,500 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:42:37,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:37,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:39,476 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:42:41,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:41,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:41,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:41,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:41,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:41,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:41,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:41,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:41,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:41,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:41,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:41,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:41,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:41,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:41,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:41,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:41,477 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:42:43,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:43,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:43,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:43,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:43,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:43,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:43,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:43,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:43,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:43,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:43,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:43,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:43,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:43,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:43,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:43,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:45,479 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:42:45,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:45,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:45,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:45,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:45,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:45,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:45,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:45,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:45,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:45,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:45,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:45,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:45,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:45,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:45,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:45,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:47,479 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:42:48,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:48,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:48,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:48,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:49,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:49,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:49,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:49,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:49,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:49,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:49,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:49,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:49,480 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:42:51,481 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:42:51,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:42:51,957 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:42:53,482 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:42:55,483 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:42:55,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:55,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:55,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:55,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:55,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:55,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:55,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:55,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:55,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:55,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:55,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:55,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:55,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:55,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:55,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:55,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:42:57,484 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:42:59,484 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:43:01,485 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:43:03,486 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:43:05,487 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:43:07,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:43:07,190 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:43:07,488 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:43:07,578 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:43:09,489 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:43:11,490 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:43:13,491 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:43:15,492 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:43:17,493 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:43:19,494 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:43:20,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:20,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:20,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:20,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:20,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:20,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:20,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:20,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:20,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:20,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:20,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:20,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:20,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:20,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:20,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:20,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:21,494 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:43:22,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:43:22,436 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:43:23,495 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:43:25,496 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:43:27,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:27,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:27,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:27,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:27,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:27,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:27,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:27,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:27,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:27,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:27,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:27,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:27,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:27,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:27,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:27,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:27,496 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:43:29,497 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:43:29,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:29,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:29,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:29,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:29,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:29,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:29,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:29,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:29,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:29,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:29,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:29,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:29,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:29,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:29,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:29,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:31,498 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:43:32,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:32,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:32,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:32,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:32,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:32,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:32,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:32,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:32,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:32,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:32,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:32,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:32,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:32,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:32,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:32,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:33,499 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:43:33,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:33,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:33,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:33,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:33,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:33,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:33,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:33,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:33,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:33,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:33,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:33,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:33,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:33,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:33,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:33,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:35,500 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:43:37,657 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:43:37,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:43:37,661 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:43:40,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:40,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:40,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:40,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:40,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:40,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:40,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:40,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:40,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:40,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:40,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:40,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:40,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:40,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:40,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:40,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:41,502 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:43:43,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:43,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:43,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:43,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:43,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:43,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:43,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:43,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:43,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:43,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:43,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:43,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:43,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:43,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:43,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:43,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:44,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:45,504 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:43:46,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:46,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:49,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:49,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:49,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:49,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:49,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:49,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:49,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:49,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:49,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:49,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:49,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:49,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:49,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:49,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:49,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:49,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:49,505 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:43:52,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:43:52,962 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:43:53,507 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:43:53,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:53,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:53,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:53,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:53,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:53,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:53,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:53,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:53,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:53,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:53,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:53,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:53,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:53,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:53,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:53,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:55,508 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:43:58,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:58,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:59,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:59,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:59,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:59,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:59,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:59,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:59,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:59,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:59,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:59,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:59,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:59,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:59,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:59,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:59,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:59,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:43:59,510 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:44:00,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:00,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:00,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:00,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:00,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:00,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:00,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:00,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:00,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:00,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:00,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:00,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:00,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:00,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:00,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:00,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:03,511 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:44:03,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:03,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:03,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:03,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:03,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:03,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:03,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:03,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:03,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:03,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:03,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:03,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:03,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:03,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:03,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:03,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:04,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:04,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:04,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:04,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:04,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:04,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:04,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:04,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:04,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:04,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:04,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:04,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:04,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:04,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:04,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:04,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:05,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:05,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:05,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:05,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:05,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:05,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:05,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:05,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:05,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:05,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:05,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:05,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:05,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:05,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:05,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:05,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:05,512 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:44:06,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:06,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:06,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:06,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:06,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:06,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:06,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:06,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:06,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:06,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:06,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:06,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:06,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:06,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:06,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:06,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:07,513 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:44:07,736 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:44:08,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:44:08,203 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:44:09,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:09,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:09,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:09,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:09,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:09,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:09,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:09,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:09,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:09,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:09,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:09,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:09,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:09,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:09,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:09,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:10,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:11,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:11,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:11,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:11,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:11,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:11,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:11,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:11,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:11,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:11,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:11,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:11,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:11,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:11,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:11,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:11,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:11,515 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:44:12,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:12,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:13,516 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:44:13,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:13,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:13,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:13,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:13,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:13,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:13,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:13,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:13,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:13,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:13,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:13,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:13,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:13,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:13,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:13,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:16,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:16,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:16,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:16,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:16,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:16,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:16,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:16,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:16,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:16,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:16,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:16,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:16,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:16,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:16,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:16,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:17,517 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:44:19,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:19,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:19,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:19,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:19,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:19,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:19,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:19,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:19,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:19,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:19,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:19,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:19,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:19,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:19,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:19,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:19,518 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:44:20,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:20,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:21,519 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:44:23,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:44:23,511 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:44:24,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:24,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,521 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:44:25,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:25,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:27,522 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:44:30,523 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:44:32,524 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:44:32,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:32,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:33,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:33,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:33,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:33,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:33,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:33,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:33,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:33,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:33,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:33,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:33,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:33,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:33,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:33,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:33,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:33,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:34,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:34,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:34,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:34,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:34,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:34,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:34,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:34,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:34,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:34,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:34,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:34,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:34,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:34,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:34,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:34,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:34,525 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:44:34,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:34,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:34,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:34,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:35,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:35,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:35,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:35,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:35,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:35,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:35,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:35,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:35,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:35,530 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 16:44:35,699 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 16:44:35,769 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 16:44:36,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:36,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:36,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:36,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:36,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:36,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:36,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:36,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:36,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:36,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:36,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:36,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:36,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:36,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:36,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:36,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:36,663 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 16:44:37,664 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:44:37,876 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:44:38,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:44:38,862 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:44:39,665 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:44:41,665 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:44:43,666 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:44:43,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:43,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:43,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:43,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:43,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:43,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:43,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:43,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:43,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:43,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:43,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:43,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:43,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:43,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:43,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:43,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:44:45,667 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:44:47,668 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:44:49,669 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:44:51,670 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:44:53,671 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:44:54,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:44:54,084 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:44:55,671 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:44:57,672 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:44:59,673 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:45:01,674 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:45:03,675 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:45:05,675 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:45:07,676 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:45:07,953 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:45:09,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:45:09,322 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:45:09,677 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:45:12,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:12,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:12,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:12,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:12,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:12,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:12,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:12,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:12,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:12,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:12,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:12,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:12,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:12,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:12,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:12,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:12,679 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:45:14,679 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:45:16,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:16,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:16,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:16,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:16,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:16,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:16,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:16,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:16,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:16,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:16,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:16,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:16,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:16,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:16,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:16,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:16,680 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:45:17,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:17,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:17,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:17,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:17,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:17,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:17,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:17,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:17,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:17,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:17,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:17,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:17,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:17,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:17,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:17,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:18,681 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:45:20,682 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:45:22,683 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:45:24,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:45:24,551 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:45:26,684 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:45:26,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:26,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:26,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:26,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:26,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:26,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:26,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:26,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:26,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:26,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:26,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:26,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:26,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:26,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:26,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:26,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:27,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:27,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:27,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:27,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:27,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:27,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:27,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:27,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:27,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:27,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:27,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:27,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:27,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:27,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:27,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:27,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,448 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,448 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:29,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:32,687 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:45:34,687 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:45:35,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:35,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:35,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:35,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:35,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:35,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:35,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:35,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:35,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:35,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:35,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:35,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:35,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:35,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:35,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:35,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:36,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:36,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:36,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:36,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:36,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:36,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:36,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:36,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:36,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:36,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:36,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:36,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:36,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:36,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:36,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:36,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:38,031 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:45:38,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:38,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:38,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:38,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:38,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:38,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:38,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:38,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:38,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:38,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:38,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:38,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:38,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:38,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:38,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:38,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:38,690 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:45:39,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:45:39,807 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:45:40,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:40,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:40,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:40,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:40,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:40,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:40,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:40,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:40,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:40,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:40,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:40,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:40,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:40,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:40,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:40,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,692 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:45:42,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:42,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:43,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:43,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:43,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:43,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:43,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:43,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:43,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:43,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:43,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:43,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:43,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:43,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:43,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:43,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:43,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:43,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:46,693 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:45:47,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:47,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:48,694 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:45:50,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:50,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:52,696 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:45:53,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:53,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:54,702 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:45:55,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:45:55,032 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:45:55,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:55,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:55,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:55,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:55,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:55,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:55,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:55,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:55,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:55,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:55,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:55,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:55,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:55,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:55,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:55,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:56,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:56,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:56,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:56,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:56,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:56,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:56,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:56,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:56,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:56,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:56,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:56,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:56,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:56,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:56,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:56,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:58,704 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:45:59,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:59,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:59,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:59,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:59,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:59,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:59,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:59,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:59,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:59,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:59,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:59,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:59,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:59,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:59,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:45:59,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:00,705 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:01,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:01,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:02,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:02,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:02,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:02,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:02,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:02,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:02,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:02,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:02,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:02,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:02,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:02,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:02,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:02,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:02,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:02,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:03,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:03,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:03,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:03,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:03,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:03,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:03,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:03,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:03,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:03,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:03,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:03,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:03,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:03,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:03,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:03,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:04,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:04,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:04,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:04,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:04,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:04,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:04,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:04,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:04,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:04,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:04,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:04,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:04,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:04,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:04,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:04,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:04,707 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:04,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:04,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:05,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:05,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:05,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:05,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:05,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:05,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:05,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:05,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:05,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:05,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:05,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:05,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:05,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:05,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:06,707 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:07,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:07,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:07,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:07,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:07,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:07,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:07,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:07,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:07,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:07,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:07,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:07,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:07,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:07,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:07,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:07,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:08,110 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:46:08,708 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:08,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:08,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:08,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:08,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:08,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:08,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:08,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:08,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:08,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:08,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:08,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:08,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:08,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:08,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:08,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:08,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:09,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:09,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:09,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:09,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:09,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:09,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:09,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:09,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:09,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:09,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:09,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:09,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:09,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:09,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:09,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:09,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:10,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:46:10,333 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:46:10,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:10,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:10,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:10,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:10,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:10,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:10,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:10,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:10,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:10,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:10,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:10,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:11,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:11,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:11,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:11,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:12,710 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:13,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:13,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:14,711 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:16,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:16,712 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:17,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:17,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:17,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:17,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:17,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:17,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:17,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:17,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:17,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:17,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:17,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:17,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:17,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:17,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:17,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:17,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:18,712 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:19,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:19,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:20,713 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:21,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:21,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:21,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:21,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:22,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:22,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:22,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:22,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:22,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:22,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:22,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:22,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:23,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:23,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:23,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:23,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:23,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:23,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:23,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:23,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:23,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:23,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:23,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:23,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:23,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:23,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:23,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:23,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:24,715 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:25,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:46:25,712 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:46:26,718 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:28,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:28,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:28,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:28,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:28,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:28,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:28,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:28,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:28,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:28,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:28,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:28,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:28,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:28,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:28,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:28,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:28,719 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:30,720 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:31,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:31,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:31,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:31,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:31,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:31,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:31,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:31,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:31,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:31,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:31,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:31,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:31,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:31,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:31,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:31,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:32,720 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:33,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:33,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:33,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:33,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:33,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:33,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:33,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:33,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:33,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:33,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:33,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:33,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:33,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:33,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:33,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:33,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:34,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:34,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:34,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:34,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:34,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:34,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:34,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:34,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:34,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:34,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:34,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:34,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:34,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:34,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:34,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:34,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:46:34,721 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:36,722 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:38,189 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:46:38,723 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:40,724 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:40,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:46:40,956 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:46:42,725 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:44,726 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:46,727 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:48,728 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:50,728 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:52,729 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:54,730 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:56,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:46:56,227 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:46:56,731 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:46:58,732 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:47:00,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:00,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:00,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:00,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:00,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:00,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:00,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:00,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:00,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:00,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:00,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:00,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:00,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:00,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:00,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:00,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:00,733 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:47:02,733 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:47:03,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:03,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:03,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:03,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:03,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:03,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:03,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:03,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:03,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:03,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:03,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:03,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:03,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:03,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:03,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:03,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:04,734 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:47:04,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:04,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:04,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:04,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:04,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:04,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:04,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:04,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:04,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:04,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:04,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:04,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:04,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:04,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:04,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:04,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:06,735 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:47:08,271 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:47:08,736 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:47:11,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:47:11,458 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:47:12,738 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:47:15,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:15,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:15,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:15,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:15,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:15,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:15,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:15,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:15,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:15,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:15,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:15,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:15,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:15,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:15,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:15,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:17,740 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:47:18,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:18,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:18,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:18,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:18,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:18,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:18,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:18,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:18,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:18,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:18,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:18,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:18,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:18,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:18,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:18,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,742 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:47:21,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:21,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:25,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:25,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:25,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:25,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:25,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:25,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:25,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:25,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:25,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:25,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:25,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:25,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:25,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:25,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:25,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:25,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:25,744 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:47:26,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:26,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:26,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:26,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:26,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:26,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:26,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:26,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:26,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:26,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:26,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:26,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:26,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:26,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:26,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:26,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:26,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:47:26,688 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:47:29,745 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:47:29,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:29,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:29,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:29,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:29,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:29,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:29,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:29,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:29,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:29,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:29,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:29,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:29,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:29,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:29,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:29,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:30,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:30,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:30,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:30,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:30,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:30,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:30,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:30,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:30,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:30,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:30,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:30,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:30,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:30,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:30,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:30,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:31,746 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:47:32,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:32,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:32,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:32,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:32,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:32,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:32,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:32,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:32,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:32,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:32,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:32,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:32,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:32,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:32,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:32,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:34,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:35,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:35,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:35,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:35,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:35,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:35,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:35,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:35,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:35,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:35,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:35,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:35,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:35,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:35,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:35,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:35,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:35,748 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:47:37,749 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:47:38,348 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:47:40,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:40,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:40,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:40,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:40,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:40,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:40,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:40,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:40,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:40,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:40,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:40,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:40,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:40,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:40,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:40,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,751 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:47:41,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:47:41,911 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:47:41,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:41,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:42,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:42,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:42,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:42,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:42,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:42,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:42,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:42,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:42,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:42,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:42,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:42,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:42,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:42,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:42,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:42,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:43,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:43,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:43,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:43,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:43,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:43,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:43,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:43,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:43,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:43,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:43,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:43,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:43,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:43,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:43,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:43,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:43,752 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:47:47,754 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:47:49,755 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:47:49,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:49,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:49,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:49,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:49,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:49,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:49,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:49,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:49,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:49,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:49,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:49,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:49,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:49,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:49,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:49,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:50,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:51,756 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:47:53,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:53,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:53,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:53,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:53,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:53,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:53,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:53,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:53,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:53,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:53,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:53,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:53,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:53,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:53,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:53,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:53,756 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:47:57,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:47:57,137 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:47:57,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,758 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:47:57,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:57,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:58,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:47:59,759 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:00,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:00,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:00,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:00,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:00,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:00,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:00,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:00,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:00,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:00,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:00,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:00,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:00,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:00,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:00,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:00,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:01,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:01,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:01,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:01,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:01,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:01,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:01,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:01,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:01,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:01,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:01,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:01,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:01,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:01,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:01,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:01,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:01,760 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:02,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:02,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:02,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:02,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:02,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:02,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:02,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:02,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:02,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:02,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:02,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:02,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:02,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:02,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:02,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:02,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:03,761 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:04,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:04,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:04,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:04,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:04,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:04,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:04,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:04,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:04,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:04,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:04,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:04,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:04,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:04,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:04,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:04,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:05,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:05,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:05,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:05,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:05,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:05,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:05,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:05,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:05,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:05,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:05,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:05,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:05,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:05,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:05,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:05,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:05,762 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:06,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:06,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:06,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:06,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:06,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:06,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:06,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:06,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:06,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:06,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:06,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:06,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:06,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:06,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:06,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:06,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:06,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:06,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:06,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:06,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:07,339 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 16:48:07,548 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 16:48:07,626 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 16:48:07,763 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:07,763 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 16:48:08,443 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:48:08,763 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:09,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:09,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:09,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:09,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:09,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:09,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:09,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:09,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:09,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:09,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:09,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:09,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:09,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:09,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:09,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:09,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:11,765 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:12,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:48:12,369 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:48:12,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:12,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:12,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:12,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:12,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:12,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:12,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:12,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:12,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:12,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:12,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:12,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:12,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:12,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:12,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:12,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:13,766 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:15,766 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:17,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:17,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:17,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:17,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:17,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:17,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:17,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:17,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:17,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:17,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:17,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:17,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:17,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:17,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:17,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:17,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:17,767 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:19,768 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:20,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:20,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:20,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:20,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:20,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:20,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:20,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:20,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:20,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:20,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:20,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:20,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:20,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:20,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:20,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:20,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:21,769 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:22,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:22,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:23,770 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:25,771 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:27,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:48:27,709 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:48:27,772 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:29,773 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:31,774 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:33,775 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:35,776 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:37,777 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:38,522 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:48:39,778 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:41,779 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:42,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:48:42,938 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:48:43,780 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:45,781 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:47,782 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:47,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:47,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:47,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:47,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:47,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:47,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:47,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:47,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:47,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:47,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:47,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:47,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:47,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:47,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:47,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:47,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:49,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:49,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:49,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:49,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:49,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:49,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:49,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:49,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:49,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:49,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:49,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:49,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:49,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:49,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:49,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:49,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:49,783 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:50,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:50,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:50,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:50,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:50,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:50,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:50,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:50,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:50,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:50,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:50,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:50,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:50,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:50,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:50,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:50,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:48:51,784 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:53,785 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:57,787 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:48:58,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:48:58,183 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:49:01,788 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:49:03,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:03,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:03,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:03,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:03,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:03,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:03,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:03,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:03,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:03,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:03,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:03,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:03,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:03,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:03,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:03,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:05,790 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:49:08,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:08,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:08,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:08,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:08,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:08,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:08,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:08,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:08,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:08,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:08,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:08,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:08,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:08,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:08,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:08,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:08,602 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:49:09,792 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:49:11,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:11,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:11,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:11,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:11,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:11,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:11,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:11,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:11,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:11,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:11,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:11,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:11,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:11,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:11,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:11,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:12,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:12,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:12,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:12,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:12,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:12,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:12,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:12,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:12,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:12,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:12,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:12,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:12,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:12,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:12,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:12,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:13,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:49:13,409 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:49:13,794 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:49:15,795 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:49:19,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:19,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:19,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:19,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:19,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:19,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:19,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:19,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:19,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:19,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:19,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:19,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:19,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:19,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:19,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:19,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,797 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:49:20,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:20,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:21,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:22,798 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:49:26,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,800 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:49:26,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:26,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:28,801 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:49:28,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:49:28,816 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:49:29,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:29,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:29,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:29,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:29,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:29,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:29,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:29,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:29,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:29,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:29,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:29,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:29,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:29,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:29,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:29,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:31,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:31,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:31,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:31,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:31,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:31,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:31,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:31,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:31,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:31,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:31,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:31,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:31,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:31,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:31,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:31,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:32,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:32,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:32,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:32,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:32,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:32,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:32,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:32,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:32,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:32,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:32,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:32,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:32,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:32,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:32,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:32,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:32,802 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:49:33,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:33,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:33,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:33,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:33,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:33,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:33,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:33,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:33,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:33,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:33,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:33,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:33,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:33,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:33,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:33,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:34,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:34,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:34,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:34,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:34,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:34,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:34,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:34,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:34,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:34,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:34,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:34,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:34,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:34,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:34,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:34,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:34,803 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:49:36,804 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:49:37,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:37,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:37,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:37,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:37,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:37,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:37,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:37,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:37,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:37,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:37,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:37,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:37,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:37,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:37,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:37,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:38,682 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:49:39,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:39,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:40,806 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:49:41,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:41,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:41,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:41,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:41,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:41,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:41,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:41,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:41,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:41,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:41,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:41,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:41,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:41,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:41,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:41,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:42,807 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:49:43,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:43,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:44,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:49:44,064 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:49:44,808 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:49:44,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:44,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:44,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:44,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:44,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:44,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:45,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:45,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:45,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:45,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:45,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:45,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:45,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:45,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:45,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:45,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:46,809 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:49:46,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:46,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:46,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:46,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:46,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:46,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:46,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:46,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:46,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:46,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:46,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:46,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:46,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:46,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:46,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:46,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:47,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:48,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:48,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:48,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:48,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:48,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:48,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:48,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:48,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:48,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:48,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:48,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:48,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:48,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:48,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:48,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:48,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:48,810 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:49:50,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:50,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:50,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:50,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:50,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:50,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:50,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:50,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:50,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:50,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:50,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:50,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:50,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:50,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:50,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:50,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:50,811 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:49:51,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:51,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:52,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:52,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:52,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:52,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:52,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:52,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:52,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:52,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:52,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:52,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:52,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:52,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:52,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:52,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:52,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:52,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:53,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:53,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:53,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:53,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:53,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:53,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:53,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:53,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:53,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:53,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:53,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:53,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:53,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:53,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:53,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:53,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,813 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:49:54,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:54,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:55,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:55,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:55,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:55,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:55,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:55,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:55,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:55,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:55,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:55,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:55,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:55,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:55,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:55,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:55,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:55,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:56,814 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:49:58,815 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:49:59,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:49:59,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:49:59,329 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:50:00,816 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:50:01,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:01,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:01,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:01,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:01,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:01,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:01,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:01,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:01,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:01,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:01,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:01,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:01,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:01,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:01,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:01,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:02,817 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:50:04,818 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:50:06,819 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:50:08,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:08,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:08,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:08,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:08,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:08,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:08,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:08,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:08,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:08,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:08,448 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:08,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:08,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:08,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:08,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:08,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:08,761 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:50:08,820 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:50:09,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:09,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:10,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:10,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:10,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:10,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:10,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:10,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:10,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:10,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:10,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:10,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:10,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:10,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:10,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:10,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:10,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:10,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:10,821 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:50:12,822 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:50:14,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:50:14,625 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:50:14,823 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:50:16,824 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:50:18,825 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:50:20,826 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:50:22,827 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:50:24,828 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:50:26,829 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:50:28,830 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:50:29,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:50:29,900 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:50:30,832 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:50:32,833 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:50:33,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:33,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:33,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:33,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:33,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:33,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:33,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:33,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:33,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:33,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:33,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:33,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:33,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:33,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:33,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:33,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:34,834 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:50:34,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:34,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:34,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:34,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:34,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:34,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:34,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:34,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:34,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:34,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:34,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:34,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:34,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:34,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:34,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:34,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:36,835 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:50:37,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:37,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:37,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:37,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:37,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:37,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:37,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:37,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:37,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:37,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:37,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:37,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:37,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:37,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:37,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:37,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:38,841 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:50:42,837 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:50:45,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:50:45,141 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:50:46,839 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:50:50,841 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:50:53,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:53,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:53,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:53,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:53,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:53,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:53,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:53,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:53,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:53,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:53,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:53,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:53,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:53,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:53,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:53,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:54,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:54,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:54,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:54,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:54,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:54,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:54,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:54,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:54,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:54,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:54,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:54,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:54,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:54,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:54,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:54,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:50:54,842 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:50:56,843 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:00,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:51:00,393 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:51:00,845 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:02,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:02,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:02,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:02,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:02,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:02,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:02,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:02,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:02,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:02,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:02,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:02,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:02,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:02,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:02,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:02,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:05,847 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:07,848 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:08,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:08,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:08,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:08,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:08,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:08,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:08,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:08,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:08,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:08,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:08,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:08,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:08,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:08,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:08,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:08,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:08,922 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:51:10,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:10,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:11,850 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:12,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:12,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:12,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:12,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:12,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:12,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:12,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:12,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:12,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:12,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:12,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:12,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:12,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:12,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:12,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:12,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:13,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:13,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:13,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:13,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:13,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:13,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:13,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:13,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:13,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:13,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:13,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:13,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:13,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:13,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:13,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:13,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:13,851 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:14,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:14,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:14,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:14,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:14,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:14,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:14,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:14,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:14,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:14,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:14,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:14,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:14,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:14,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:14,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:14,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:15,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:51:15,618 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:51:15,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:15,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:15,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:15,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:15,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:15,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:15,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:15,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:15,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:15,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:15,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:15,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:15,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:15,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:15,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:15,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:17,853 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:18,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:18,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:19,854 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:19,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:19,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:19,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:19,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:19,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:19,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:19,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:19,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:19,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:19,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:19,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:19,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:19,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:19,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:19,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:19,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:20,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:20,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:20,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:20,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:20,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:20,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:20,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:20,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:20,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:20,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:20,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:20,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:20,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:20,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:20,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:20,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:21,855 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:22,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:22,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:22,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:22,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:22,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:22,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:22,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:22,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:22,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:22,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:22,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:22,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:22,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:22,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:22,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:22,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:23,856 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:24,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:24,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:24,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:24,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:24,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:24,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:24,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:24,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:24,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:24,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:24,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:24,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:24,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:24,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:24,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:24,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:25,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:25,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:25,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:25,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:25,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:25,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:25,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:25,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:25,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:25,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:25,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:25,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:25,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:25,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:25,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:25,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:26,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:26,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:26,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:26,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:26,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:26,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:26,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:26,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:26,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:26,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:26,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:26,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:26,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:26,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:26,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:26,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:27,858 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:28,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:28,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:28,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:28,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:28,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:28,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:28,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:28,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:28,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:28,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:28,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:28,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:28,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:28,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:28,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:28,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:29,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:29,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:29,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:29,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:29,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:29,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:29,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:29,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:29,859 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:29,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:29,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:29,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:29,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:29,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:29,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:29,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:29,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:30,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:30,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:30,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:30,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:30,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:30,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:30,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:30,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:30,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:30,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:30,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:30,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:30,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:30,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:30,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:30,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:30,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:51:30,842 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:51:31,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:31,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:31,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:31,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:31,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:31,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:31,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:31,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:31,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:31,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:31,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:31,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:31,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:31,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:31,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:31,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:31,860 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:32,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:32,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:33,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:33,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:33,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:33,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:33,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:33,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:33,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:33,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:33,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:33,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:33,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:33,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:33,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:33,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:33,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:33,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:33,861 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:35,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,862 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:35,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:35,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:36,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:36,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:36,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:36,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:36,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:36,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:36,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:36,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:36,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:36,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:36,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:36,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:36,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:36,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:36,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:36,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:36,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:36,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:36,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:36,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:37,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:37,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:37,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:37,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:37,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:37,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:37,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:37,074 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:37,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:37,341 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 16:51:37,562 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 16:51:37,636 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 16:51:37,863 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 16:51:37,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:37,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:37,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:37,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:37,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:37,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:37,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:37,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:37,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:37,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:37,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:37,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:37,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:37,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:37,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:37,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:38,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:38,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:38,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:38,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:38,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:38,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:38,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:38,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:38,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:38,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:38,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:38,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:38,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:38,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:38,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:38,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:38,863 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:39,040 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:51:39,864 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:41,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:41,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:41,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:41,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:41,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:41,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:41,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:41,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:41,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:41,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:41,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:41,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:41,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:41,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:41,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:41,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:41,865 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:42,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:42,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:42,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:42,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:42,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:42,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:42,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:42,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:42,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:42,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:42,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:42,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:42,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:42,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:42,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:42,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:43,866 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:44,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:44,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:44,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:44,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:44,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:44,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:44,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:44,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:44,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:44,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:44,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:44,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:44,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:44,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:44,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:44,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:45,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:45,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:45,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:45,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:45,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:45,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:45,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:45,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:45,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:45,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:45,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:45,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:45,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:45,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:45,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:45,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:45,867 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:46,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:51:46,145 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:51:46,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:46,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:47,868 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:48,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:48,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:48,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:48,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:48,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:48,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:48,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:48,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:48,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:48,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:48,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:48,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:48,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:48,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:48,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:48,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:49,869 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:51,869 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:53,870 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:54,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:54,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:54,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:54,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:54,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:54,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:54,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:54,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:54,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:54,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:54,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:54,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:54,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:54,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:54,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:54,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:55,871 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:57,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:57,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:57,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:57,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:57,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:57,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:57,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:57,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:57,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:57,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:57,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:57,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:57,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:57,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:57,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:57,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:57,872 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:51:59,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:59,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:59,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:59,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:59,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:59,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:59,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:59,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:59,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:59,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:59,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:59,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:59,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:59,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:59,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:59,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:51:59,873 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:52:00,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:00,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:00,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:00,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:00,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:00,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:00,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:00,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:00,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:00,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:00,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:00,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:00,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:00,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:00,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:00,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:01,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:52:01,392 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:52:01,876 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:52:03,877 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:52:05,878 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:52:07,879 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:52:09,120 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:52:09,879 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:52:11,880 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:52:13,881 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:52:15,882 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:52:16,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:52:16,617 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:52:17,883 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:52:19,884 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:52:21,885 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:52:23,886 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:52:24,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:24,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:24,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:24,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:24,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:24,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:24,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:24,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:24,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:24,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:24,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:24,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:24,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:24,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:24,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:24,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:29,889 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:52:30,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:30,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:30,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:30,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:30,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:30,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:30,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:30,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:30,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:30,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:30,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:30,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:30,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:30,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:30,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:30,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:32,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:52:32,032 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:52:32,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:32,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:32,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:32,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:32,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:32,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:32,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:32,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:32,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:32,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:32,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:32,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:32,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:32,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:32,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:32,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:33,890 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:52:35,891 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:52:39,199 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:52:40,893 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:52:44,895 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:52:45,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:45,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:45,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:45,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:45,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:45,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:45,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:45,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:45,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:45,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:45,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:45,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:45,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:45,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:45,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:45,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:47,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:52:47,276 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:52:48,897 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:52:50,898 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:52:51,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:51,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:51,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:51,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:51,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:51,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:51,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:51,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:51,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:51,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:51,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:51,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:51,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:51,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:51,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:51,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:53,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:53,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:53,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:53,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:53,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:53,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:53,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:53,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:53,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:53,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:53,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:53,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:53,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:53,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:53,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:53,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:54,900 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:52:56,901 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:52:57,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:57,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:57,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:57,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:57,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:57,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:57,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:57,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:57,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:57,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:57,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:57,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:57,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:57,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:57,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:57,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:52:58,902 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:00,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:00,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:01,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:01,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:01,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:01,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:01,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:01,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:01,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:01,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:01,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:01,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:01,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:01,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:01,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:01,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:01,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:01,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:02,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:53:02,499 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:53:02,904 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:02,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:02,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:02,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:02,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:02,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:02,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:02,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:02,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:02,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:02,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:02,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:02,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:02,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:02,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:02,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:02,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:04,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:04,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:04,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:04,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:04,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:04,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:04,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:04,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:04,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:04,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:04,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:04,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:04,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:04,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:04,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:04,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:04,905 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:06,906 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:06,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:06,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:06,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:06,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:06,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:06,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:06,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:06,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:06,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:06,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:06,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:06,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:06,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:06,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:06,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:06,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:08,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:08,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:08,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:08,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:08,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:08,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:08,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:08,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:08,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:08,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:08,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:08,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:08,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:08,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:08,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:08,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:09,279 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:53:10,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:10,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:10,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:10,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:10,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:10,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:10,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:10,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:10,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:10,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:10,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:10,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:10,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:10,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:10,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:10,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:10,908 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:11,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:11,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:11,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:11,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:11,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:11,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:11,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:11,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:11,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:11,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:11,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:11,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:11,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:11,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:11,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:11,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:12,909 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:14,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:14,910 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:15,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:15,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:15,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:15,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:15,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:15,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:15,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:15,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:15,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:15,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:15,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:15,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:15,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:15,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:15,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:15,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:16,911 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:17,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:53:17,728 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:53:18,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:18,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:18,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:18,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:18,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:18,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:18,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:18,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:18,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:18,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:18,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:18,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:18,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:18,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:18,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:18,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:18,912 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:19,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:19,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:21,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:21,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:21,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:21,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:21,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:21,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:21,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:21,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:21,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:21,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:21,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:21,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:21,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:21,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:21,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:21,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:21,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:21,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:21,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:21,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:22,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:22,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:22,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:22,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:22,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:22,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:22,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:22,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:22,913 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:23,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,400 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:23,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:24,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:24,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:24,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:24,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:24,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:24,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:24,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:24,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:24,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:24,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:24,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:24,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:24,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:24,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:24,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:24,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:24,914 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:25,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:25,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:26,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:26,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:26,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:26,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:26,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:26,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:26,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:26,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:26,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:26,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:26,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:26,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:26,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:26,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:26,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:26,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:26,916 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:27,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:27,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:27,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:27,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:27,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:27,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:27,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:27,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:27,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:27,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:27,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:27,871 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:27,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:27,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:27,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:27,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:28,916 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:28,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:28,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:28,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:28,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:28,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:28,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:28,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:28,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:28,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:28,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:28,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:28,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:28,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:28,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:28,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:28,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:29,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:29,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:29,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:29,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:29,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:29,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:29,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:29,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:29,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:29,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:29,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:29,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:29,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:29,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:29,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:29,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:30,917 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:32,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:32,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:32,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:32,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:32,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:32,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:32,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:32,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:32,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:32,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:32,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:32,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:32,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:32,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:32,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:32,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:32,918 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:32,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:53:32,973 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:53:33,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:33,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:34,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:34,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:34,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:34,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:34,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:34,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:34,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:34,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:34,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:34,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:34,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:34,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:34,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:34,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:34,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:34,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:34,919 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:36,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:36,919 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:37,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:37,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:37,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:37,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:37,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:37,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:37,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:37,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:37,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:37,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:37,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:37,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:37,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:37,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:37,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:37,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:38,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:38,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:38,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:38,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:38,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:38,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:38,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:38,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:38,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:38,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:38,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:38,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:38,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:38,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:38,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:38,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:38,920 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:39,358 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:53:40,921 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:42,922 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:43,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:43,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:43,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:43,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:43,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:43,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:43,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:43,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:43,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:43,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:43,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:43,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:43,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:43,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:43,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:43,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:44,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:44,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:44,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:44,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:44,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:44,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:44,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:44,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:44,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:44,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:44,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:44,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:44,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:44,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:44,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:44,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:44,923 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:46,924 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:48,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:53:48,207 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:53:48,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:48,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:48,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:48,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:48,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:48,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:48,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:48,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:48,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:48,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:48,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:48,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:48,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:48,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:48,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:48,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:48,925 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:50,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:50,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:50,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:50,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:50,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:50,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:50,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:50,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:50,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:50,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:50,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:50,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:50,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:50,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:50,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:50,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:53:50,926 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:52,927 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:54,928 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:56,929 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:53:58,930 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:54:00,931 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:54:02,932 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:54:03,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:54:03,437 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:54:04,933 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:54:06,934 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:54:08,935 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:54:09,438 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:54:12,937 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:54:13,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:13,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:13,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:13,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:13,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:13,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:13,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:13,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:13,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:13,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:13,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:13,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:13,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:13,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:13,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:13,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:16,939 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:54:18,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:54:18,697 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:54:19,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:19,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:19,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:19,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:19,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:19,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:19,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:19,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:19,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:19,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:19,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:19,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:19,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:19,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:19,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:19,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:20,940 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:54:21,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:21,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:21,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:21,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:21,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:21,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:21,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:21,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:21,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:21,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:21,080 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:21,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:21,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:21,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:21,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:21,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:24,942 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:54:28,944 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:54:30,945 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:54:34,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:54:34,012 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:54:34,947 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:54:38,948 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:54:39,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,517 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:54:39,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:39,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:40,949 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:54:42,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:42,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:42,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:42,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:42,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:42,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:42,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:42,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:42,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:42,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:42,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:42,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:42,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:42,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:42,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:42,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:44,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:44,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:44,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:44,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:44,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:44,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:44,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:44,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:44,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:44,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:44,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:44,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:44,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:44,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:44,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:44,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:44,951 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:54:46,952 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:54:47,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:47,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:47,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:47,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:47,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:47,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:47,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:47,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:47,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:47,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:47,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:47,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:47,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:47,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:47,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:47,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:48,953 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:54:49,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:54:49,239 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:54:50,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:50,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:50,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:50,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:50,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:50,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:50,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:50,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:50,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:50,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:50,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:50,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:50,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:50,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:50,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:50,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:52,955 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:54:53,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:53,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:53,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:53,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:53,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:53,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:53,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:53,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:53,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:53,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:53,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:53,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:53,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:53,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:53,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:53,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:54,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:54,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:54,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:54,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:54,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:54,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:54,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:54,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:54,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:54,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:54,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:54,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:54,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:54,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:54,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:54,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:54,956 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:54:56,957 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:54:59,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,692 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:54:59,958 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:03,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:03,960 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:04,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:55:04,471 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:55:04,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:04,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:05,961 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:06,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:06,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:07,962 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:08,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:08,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:08,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:08,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:08,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:08,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:08,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:08,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:08,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 16:55:08,277 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 16:55:08,488 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 16:55:08,565 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 16:55:08,962 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 16:55:09,669 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:55:09,963 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:11,964 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:13,965 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:15,966 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:17,967 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:19,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:55:19,767 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:55:19,968 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:21,969 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:23,970 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:25,971 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:27,972 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:29,973 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:31,974 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:33,975 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:35,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:55:35,063 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:55:35,976 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:37,977 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:39,747 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:55:39,978 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:41,979 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:43,980 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:45,981 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:47,982 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:49,983 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:50,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:55:50,290 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:55:51,984 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:53,985 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:55,986 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:57,987 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:55:59,988 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:01,989 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:03,990 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:05,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:56:05,602 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:56:05,992 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:07,993 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:09,826 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:56:09,994 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:11,995 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:13,996 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:16,997 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:18,998 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:20,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:56:20,876 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:56:20,999 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:23,000 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:25,001 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:27,002 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:29,003 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:31,004 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:33,005 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:35,006 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:36,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:56:36,212 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:56:37,007 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:39,008 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:39,904 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:56:41,009 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:43,010 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:45,011 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:47,012 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:49,013 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:51,014 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:51,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:56:51,437 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:56:53,015 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:55,017 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:57,018 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:56:59,019 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:01,020 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:03,021 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:05,022 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:06,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:57:06,736 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:57:07,023 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:09,024 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:09,984 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:57:11,025 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:13,026 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:15,027 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:17,028 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:19,029 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:21,030 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:21,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:57:21,989 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:57:23,031 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:25,032 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:27,033 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:29,034 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:31,035 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:33,036 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:35,037 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:37,038 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:37,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:57:37,283 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:57:39,039 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:40,062 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:57:41,040 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:43,041 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:46,042 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:48,043 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:50,044 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:52,045 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:52,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:57:52,558 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:57:54,046 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:56,047 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:57:58,048 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:00,049 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:02,051 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:04,052 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:06,053 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:07,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:58:07,814 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:58:08,054 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:10,055 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:10,141 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:58:12,056 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:14,057 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:16,058 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:18,059 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:20,060 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:22,061 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:23,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:58:23,098 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:58:24,062 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:26,063 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:28,064 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:30,065 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:32,066 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:35,067 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:37,068 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:38,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:58:38,427 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:58:39,069 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:40,219 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:58:41,070 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:43,071 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:45,072 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:47,073 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:49,074 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:51,075 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:53,076 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:53,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:58:53,654 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:58:55,077 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:57,078 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:58:59,079 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:01,080 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:03,081 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:05,083 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:07,083 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:08,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:59:08,953 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:59:09,084 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:10,299 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:59:11,085 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:13,086 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:15,087 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:17,088 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:19,089 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:21,090 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:23,091 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:24,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:59:24,239 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:59:25,092 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:27,093 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:29,094 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:31,095 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:33,096 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:35,097 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:37,098 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:39,099 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:39,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:59:39,574 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:59:40,379 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 16:59:41,101 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:43,101 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:45,102 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:47,103 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:50,105 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:52,106 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:54,107 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:54,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 16:59:54,808 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 16:59:56,108 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 16:59:58,109 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:00,110 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:02,111 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:04,112 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:06,113 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:08,114 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:10,115 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:10,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:00:10,170 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:00:10,458 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:00:12,116 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:14,117 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:16,118 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:18,119 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:20,120 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:22,121 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:24,122 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:25,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:00:25,408 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:00:26,123 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:28,124 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:30,125 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:32,126 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:34,127 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:36,128 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:38,129 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:40,130 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:40,536 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:00:40,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:00:40,637 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:00:42,131 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:44,132 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:46,133 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:48,134 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:50,136 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:52,137 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:54,137 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:55,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:00:55,943 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:00:56,139 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:00:58,140 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:00,140 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:02,141 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:05,143 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:07,144 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:09,145 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:10,615 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:01:11,146 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:11,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:01:11,208 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:01:13,147 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:15,148 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:17,149 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:19,150 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:21,151 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:23,152 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:25,153 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:26,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:01:26,441 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:01:27,154 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:29,155 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:31,156 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:33,157 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:35,158 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:37,159 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:39,160 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:40,696 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:01:41,161 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:41,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:01:41,788 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:01:43,162 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:45,163 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:47,164 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:49,165 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:51,166 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:53,167 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:55,168 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:57,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:01:57,037 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:01:57,169 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:01:59,169 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:01,170 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:03,171 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:05,172 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:07,173 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:09,174 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:10,778 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:02:11,175 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:12,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:02:12,304 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:02:13,176 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:15,177 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:17,178 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:19,179 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:22,181 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:24,182 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:26,183 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:27,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:02:27,580 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:02:28,184 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:30,185 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:32,186 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:34,187 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:36,188 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:38,189 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:40,190 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:40,857 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:02:42,191 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:42,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:02:42,962 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:02:44,193 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:46,194 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:48,195 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:50,196 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:52,197 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:54,198 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:56,199 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:58,200 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:02:58,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:02:58,225 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:03:00,200 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:02,201 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:04,202 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:06,204 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:08,205 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:10,206 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:10,935 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:03:12,207 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:13,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:03:13,446 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:03:14,208 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:16,208 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:18,209 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:20,210 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:22,211 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:24,212 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:26,213 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:28,214 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:28,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:03:28,736 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:03:30,215 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:32,216 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:34,217 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:37,219 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:39,220 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:41,015 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:03:41,221 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:43,222 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:43,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:03:43,958 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:03:45,223 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:47,224 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:49,225 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:51,226 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:53,227 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:55,228 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:57,229 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:59,230 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:03:59,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:03:59,357 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:04:01,231 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:03,232 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:05,233 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:07,234 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:09,235 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:11,094 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:04:11,236 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:13,237 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:14,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:04:14,589 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:04:15,238 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:17,239 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:19,240 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:21,241 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:23,242 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:25,243 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:27,244 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:29,245 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:29,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:04:29,903 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:04:31,246 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:33,247 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:35,248 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:37,249 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:39,250 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:41,174 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:04:41,251 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:43,252 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:45,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:04:45,128 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:04:45,253 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:47,254 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:49,255 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:51,256 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:53,257 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:55,258 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:04:57,259 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:00,260 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:00,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:05:00,355 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:05:02,262 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:04,263 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:06,264 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:08,265 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:10,266 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:11,253 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:05:12,267 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:14,268 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:15,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:05:15,642 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:05:16,269 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:18,270 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:20,271 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:22,272 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:24,273 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:26,274 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:28,275 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:30,276 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:30,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:05:30,944 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:05:32,277 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:34,278 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:36,279 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:38,280 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:40,281 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:41,332 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:05:42,282 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:44,283 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:46,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:05:46,255 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:05:46,284 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:48,285 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:50,286 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:52,287 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:54,288 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:56,289 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:05:58,290 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:00,291 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:01,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:06:01,525 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:06:02,292 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:04,293 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:06,294 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:08,295 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:10,296 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:11,412 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:06:12,297 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:14,298 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:16,299 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:16,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:06:16,817 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:06:19,301 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:21,302 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:23,303 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:25,304 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:27,305 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:29,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:31,307 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:32,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:06:32,117 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:06:33,308 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:35,309 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:37,310 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:39,311 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:41,312 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:41,490 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:06:43,313 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:45,314 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:47,315 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:47,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:06:47,343 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:06:49,316 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:51,317 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:53,318 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:55,319 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:57,320 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:06:59,321 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:01,322 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:02,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:07:02,616 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:07:03,323 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:05,324 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:07,325 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:09,326 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:11,327 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:11,569 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:07:13,328 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:15,329 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:17,330 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:17,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:07:17,869 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:07:19,331 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:21,332 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:23,333 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:25,334 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:27,335 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:29,336 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:31,337 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:33,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:07:33,227 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:07:33,338 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:35,339 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:37,340 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:39,341 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:41,342 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:41,648 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:07:43,343 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:45,344 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:47,345 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:48,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:07:48,454 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:07:49,346 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:51,347 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:53,348 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:55,349 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:57,350 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:07:59,351 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:01,352 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:03,353 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:03,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:08:03,792 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:08:05,354 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:07,355 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:09,356 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:11,357 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:11,727 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:08:13,358 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:15,359 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:17,359 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:19,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:08:19,018 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:08:19,360 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:22,361 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:24,362 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:26,363 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:28,363 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:30,364 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:32,365 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:34,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:08:34,324 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:08:34,366 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:36,367 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:38,368 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:40,368 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:41,807 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:08:42,369 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:44,370 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:46,371 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:48,371 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:49,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:08:49,638 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:08:50,372 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:52,373 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:54,373 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:56,374 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:08:58,375 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:00,376 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:02,377 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:04,377 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:04,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:09:04,879 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:09:06,378 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:08,379 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:10,380 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:11,885 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:09:12,381 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:14,381 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:16,382 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:18,383 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:20,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:09:20,137 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:09:20,384 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:22,385 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:24,386 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:26,387 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:28,388 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:30,389 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:32,389 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:34,390 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:35,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:09:35,415 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:09:36,391 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:38,392 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:40,393 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:41,964 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:09:42,394 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:44,395 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:47,396 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:49,396 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:50,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:09:50,713 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:09:51,397 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:53,398 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:55,399 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:57,400 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:09:59,401 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:01,401 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:03,402 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:05,403 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:06,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:10:06,030 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:10:07,404 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:09,405 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:11,405 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:12,041 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:10:13,406 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:15,407 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:17,408 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:19,409 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:21,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:10:21,260 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:10:21,410 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:23,411 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:25,412 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:27,413 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:29,413 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:31,414 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:33,415 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:35,416 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:36,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:10:36,510 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:10:37,417 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:39,418 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:41,419 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:42,122 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:10:43,420 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:45,420 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:47,421 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:49,422 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:51,423 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:51,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:10:51,778 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:10:53,424 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:55,425 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:57,425 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:10:59,426 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:01,427 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:03,428 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:05,429 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:07,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:11:07,003 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:11:07,430 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:10,431 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:12,201 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:11:12,432 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:14,433 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:16,434 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:18,435 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:20,436 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:22,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:11:22,228 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:11:22,437 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:24,438 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:26,438 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:28,439 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:30,440 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:32,440 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:34,441 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:36,442 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:37,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:11:37,528 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:11:38,443 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:40,444 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:42,281 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:11:42,445 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:44,445 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:46,446 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:48,447 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:50,448 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:52,449 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:52,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:11:52,795 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:11:54,449 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:56,450 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:11:58,451 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:00,452 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:02,452 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:04,453 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:06,454 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:08,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:12:08,027 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:12:08,455 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:11,457 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:12,361 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:12:13,458 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:15,459 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:17,460 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:19,461 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:21,462 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:23,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:12:23,399 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:12:23,463 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:25,464 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:27,465 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:29,466 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:31,467 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:33,468 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:35,468 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:37,469 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:38,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:12:38,622 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:12:39,471 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:41,472 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:42,441 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:12:43,472 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:45,473 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:47,474 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:49,475 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:52,477 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:53,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:12:53,994 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:12:54,477 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:56,478 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:12:58,479 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:00,480 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:02,481 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:04,482 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:06,483 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:08,484 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:09,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:13:09,282 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:13:10,485 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:12,486 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:12,520 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:13:14,487 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:16,488 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:18,489 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:20,490 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:22,490 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:24,491 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:24,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:13:24,669 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:13:26,492 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:28,493 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:30,494 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:32,495 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:34,496 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:36,496 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:38,497 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:39,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:13:39,966 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:13:40,498 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:42,499 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:42,599 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:13:44,500 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:46,500 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:48,501 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:50,502 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:52,503 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:54,504 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:55,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:13:55,191 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:13:56,505 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:13:58,506 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:00,507 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:02,508 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:04,509 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:06,509 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:09,511 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:10,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:14:10,501 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:14:11,512 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:12,680 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:14:13,513 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:15,513 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:17,514 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:19,515 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:21,516 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:23,517 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:25,518 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:25,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:14:25,805 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:14:27,519 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:29,520 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:31,521 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:33,522 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:35,522 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:37,523 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:39,524 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:41,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:14:41,061 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:14:41,525 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:42,759 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:14:43,526 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:45,527 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:47,528 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:49,529 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:51,530 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:53,530 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:55,531 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:56,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:14:56,285 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:14:57,532 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:14:59,533 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:01,534 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:03,535 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:05,536 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:07,537 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:09,538 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:11,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:15:11,525 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:15:11,539 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:12,837 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:15:13,540 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:15,541 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:18,542 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:20,543 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:22,544 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:24,545 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:26,546 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:26,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:15:26,825 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:15:28,547 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:30,548 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:32,549 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:34,549 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:36,550 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:38,551 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:40,552 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:42,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:15:42,142 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:15:42,553 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:42,914 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:15:44,554 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:46,554 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:48,555 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:50,556 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:52,557 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:54,558 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:56,559 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:15:57,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:15:57,369 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:15:58,560 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:00,561 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:02,562 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:04,563 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:06,564 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:08,564 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:10,565 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:12,566 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:12,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:16:12,719 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:16:12,990 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:16:14,567 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:16,568 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:18,569 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:20,570 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:22,571 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:24,572 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:26,573 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:27,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:16:27,964 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:16:28,574 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:30,575 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:32,576 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:34,577 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:36,578 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:38,579 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:40,580 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:42,581 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:43,066 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:16:43,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:16:43,345 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:16:44,582 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:46,583 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:48,584 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:50,584 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:52,585 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:54,586 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:56,587 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:16:58,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:16:58,571 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:16:58,587 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:00,588 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:02,589 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:04,589 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:06,590 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:09,591 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:11,592 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:13,143 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:17:13,593 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:13,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:17:13,795 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:17:15,594 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:17,595 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:19,596 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:21,597 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:23,597 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:25,598 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:27,599 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:29,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:17:29,084 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:17:29,600 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:31,601 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:33,602 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:35,603 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:37,604 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:39,605 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:41,606 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:43,223 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:17:43,607 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:44,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:17:44,314 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:17:45,608 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:47,609 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:49,610 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:51,611 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:53,611 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:55,612 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:57,613 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:17:59,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:17:59,537 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:17:59,614 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:01,615 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:03,616 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:05,616 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:07,617 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:09,618 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:11,619 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:13,301 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:18:13,619 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:14,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:18:14,844 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:18:15,620 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:17,621 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:19,622 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:21,623 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:24,624 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:26,625 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:28,626 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:30,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:18:30,125 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:18:30,627 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:32,628 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:34,628 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:36,629 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:38,630 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:40,631 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:42,632 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:43,380 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:18:44,633 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:45,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:18:45,414 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:18:46,634 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:48,635 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:50,636 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:52,636 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:54,637 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:56,638 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:18:58,639 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:00,640 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:00,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:19:00,710 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:19:02,641 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:04,642 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:06,643 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:08,644 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:10,645 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:12,646 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:13,457 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:19:14,647 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:16,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:19:16,018 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:19:16,648 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:18,650 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:20,651 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:22,652 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:24,653 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:26,654 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:28,654 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:30,655 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:31,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:19:31,335 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:19:32,656 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:34,657 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:36,658 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:38,659 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:40,660 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:42,661 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:43,536 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:19:44,662 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:46,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:19:46,568 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:19:46,663 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:48,664 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:50,665 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:52,666 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:54,667 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:56,668 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:19:58,668 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:00,669 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:01,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:20:01,796 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:20:02,670 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:05,672 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:07,672 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:09,673 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:11,674 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:13,614 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:20:13,675 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:15,676 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:17,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:20:17,023 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:20:17,677 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:19,678 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:21,679 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:23,679 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:25,680 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:27,681 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:29,682 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:31,683 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:32,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:20:32,334 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:20:33,684 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:35,685 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:37,686 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:39,687 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:41,688 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:43,689 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:43,693 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:20:45,690 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:47,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:20:47,649 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:20:47,691 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:49,692 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:51,693 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:53,694 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:55,695 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:57,696 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:20:59,697 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:01,698 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:02,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:21:02,916 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:21:03,699 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:05,700 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:07,701 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:09,702 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:11,703 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:13,704 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:13,772 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:21:15,705 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:17,706 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:18,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:21:18,141 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:21:19,707 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:21,708 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:23,709 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:25,710 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:27,711 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:29,712 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:31,713 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:33,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:21:33,399 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:21:34,714 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:36,715 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:38,716 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:40,717 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:42,719 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:43,850 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:21:44,720 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:46,721 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:48,721 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:48,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:21:48,732 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:21:50,723 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:52,724 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:54,725 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:56,726 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:21:58,727 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:00,728 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:02,728 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:04,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:22:04,042 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:22:04,729 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:06,730 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:08,731 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:10,732 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:12,734 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:13,929 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:22:14,735 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:16,735 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:18,736 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:19,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:22:19,315 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:22:20,737 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:22,738 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:24,739 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:26,740 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:28,741 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:30,742 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:32,743 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:34,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:22:34,550 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:22:34,744 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:36,745 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:39,747 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:41,748 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:43,749 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:44,008 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:22:45,750 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:47,751 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:49,751 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:49,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:22:49,952 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:22:51,752 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:53,753 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:55,754 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:57,754 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:22:59,755 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:01,756 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:03,757 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:05,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:23:05,251 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:23:05,758 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:07,758 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:09,759 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:11,760 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:13,760 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:14,086 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:23:15,761 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:17,762 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:19,763 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:20,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:23:20,477 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:23:21,763 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:23,764 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:25,765 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:27,766 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:29,767 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:31,767 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:33,768 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:35,769 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:35,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:23:35,826 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:23:37,770 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:39,770 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:41,771 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:43,772 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:44,166 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:23:45,773 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:47,773 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:49,774 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:51,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:23:51,051 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:23:51,775 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:53,776 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:55,776 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:57,777 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:23:59,778 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:01,778 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:03,779 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:05,780 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:06,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:24:06,611 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:24:07,781 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:10,782 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:12,783 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:14,246 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:24:14,783 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:16,784 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:18,785 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:20,786 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:21,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:24:21,840 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:24:22,786 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:24,787 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:26,788 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:28,788 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:30,789 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:32,790 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:34,791 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:36,791 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:37,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:24:37,196 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:24:38,792 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:40,793 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:42,794 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:44,324 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:24:44,794 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:46,795 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:48,796 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:50,797 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:52,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:24:52,417 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:24:52,798 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:54,798 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:56,799 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:24:58,800 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:01,801 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:03,802 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:05,802 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:07,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:25:07,721 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:25:07,803 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:09,804 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:11,805 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:13,806 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:14,403 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:25:15,806 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:17,807 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:19,808 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:21,808 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:22,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:25:22,946 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:25:23,809 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:25,810 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:27,811 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:29,811 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:31,812 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:33,813 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:35,814 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:37,814 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:38,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:25:38,332 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:25:39,815 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:41,816 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:43,817 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:44,483 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:25:45,817 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:47,818 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:49,819 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:51,820 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:53,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:25:53,557 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:25:53,820 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:55,821 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:57,822 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:25:59,823 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:01,824 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:03,824 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:05,825 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:07,826 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:08,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:26:08,862 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:26:09,827 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:11,828 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:13,828 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:14,563 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:26:15,829 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:17,830 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:19,831 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:21,831 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:24,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:26:24,096 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:26:24,832 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:26,833 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:28,834 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:30,835 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:32,835 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:34,836 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:36,837 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:38,837 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:39,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:26:39,367 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:26:40,838 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:42,839 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:44,642 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:26:44,840 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:46,841 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:48,841 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:50,842 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:52,843 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:54,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:26:54,599 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:26:54,844 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:56,844 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:26:58,845 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:00,846 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:02,846 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:04,847 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:06,848 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:08,849 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:09,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:27:09,925 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:27:10,849 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:12,850 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:14,719 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:27:14,851 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:16,852 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:18,852 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:20,853 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:22,854 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:24,855 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:25,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:27:25,151 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:27:26,856 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:28,857 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:30,858 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:32,858 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:34,859 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:36,860 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:38,861 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:40,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:27:40,491 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:27:40,862 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:42,863 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:44,798 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:27:44,864 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:46,865 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:48,866 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:50,866 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:53,867 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:55,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:27:55,784 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:27:55,868 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:57,869 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:27:59,870 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:01,870 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:03,871 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:05,872 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:07,873 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:09,873 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:11,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:28:11,106 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:28:11,874 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:13,875 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:14,877 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:28:15,876 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:17,877 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:19,877 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:21,878 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:23,879 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:25,880 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:26,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:28:26,335 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:28:27,880 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:29,881 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:31,882 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:33,883 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:35,883 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:37,884 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:39,885 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:41,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:28:41,706 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:28:41,886 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:43,887 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:44,957 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:28:45,888 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:47,889 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:49,890 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:51,890 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:53,891 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:55,892 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:57,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:28:57,189 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:28:57,893 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:28:59,894 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:01,895 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:03,895 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:05,896 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:08,897 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:10,898 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:12,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:29:12,467 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:29:12,899 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:14,900 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:15,036 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:29:16,901 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:18,901 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:20,902 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:22,903 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:24,904 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:26,905 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:27,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:29:27,757 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:29:28,906 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:30,906 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:32,907 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:34,908 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:36,909 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:38,909 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:40,910 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:42,911 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:43,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:29:43,036 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:29:44,912 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:45,114 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:29:46,913 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:48,913 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:50,914 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:52,915 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:54,916 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:56,917 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:29:58,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:29:58,349 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:29:58,918 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:00,918 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:02,919 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:04,920 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:06,921 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:09,922 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:11,922 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:13,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:30:13,616 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:30:13,923 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:15,193 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:30:15,924 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:17,925 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:19,925 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:21,926 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:23,927 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:25,928 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:27,928 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:28,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:30:28,864 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:30:29,929 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:31,930 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:33,931 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:35,932 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:37,932 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:39,933 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:41,934 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:43,934 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:44,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:30:44,093 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:30:45,272 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:30:45,935 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:47,936 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:49,937 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:51,937 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:53,938 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:55,939 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:57,940 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:30:59,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:30:59,320 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:30:59,940 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:01,941 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:03,942 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:05,943 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:07,944 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:09,945 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:11,946 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:13,946 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:14,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:31:14,561 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:31:15,351 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:31:15,947 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:17,948 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:19,949 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:21,950 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:23,951 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:25,951 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:27,952 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:29,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:31:29,789 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:31:30,953 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:32,954 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:34,955 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:36,956 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:38,956 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:40,957 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:42,958 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:44,959 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:45,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:31:45,073 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:31:45,431 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:31:46,960 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:48,960 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:50,961 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:52,962 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:54,963 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:56,964 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:31:58,965 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:00,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:32:00,365 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:32:00,966 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:02,966 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:04,967 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:06,968 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:08,969 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:10,970 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:12,971 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:14,971 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:15,511 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:32:15,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:32:15,593 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:32:16,972 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:18,973 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:20,974 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:22,975 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:24,976 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:26,976 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:28,977 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:30,978 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:31,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:32:31,327 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:32:32,979 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:34,979 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:36,980 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:38,981 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:40,982 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:42,983 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:44,984 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:45,589 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:32:46,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:32:46,600 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:32:46,985 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:48,986 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:50,986 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:52,987 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:54,988 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:56,989 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:32:58,990 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:00,990 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:01,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:33:01,825 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:33:02,991 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:04,992 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:06,993 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:08,994 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:10,995 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:12,995 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:14,996 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:15,668 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:33:16,997 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:17,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:33:17,116 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:33:18,998 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:21,999 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:24,000 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:26,000 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:28,001 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:30,002 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:32,003 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:32,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:33:32,370 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:33:34,004 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:36,004 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:38,005 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:40,006 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:42,007 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:44,008 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:45,749 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:33:46,009 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:47,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:33:47,622 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:33:48,010 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:50,010 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:52,011 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:54,012 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:56,013 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:33:58,013 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:00,014 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:02,015 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:02,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:34:02,849 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:34:04,016 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:06,017 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:08,018 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:10,020 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:12,020 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:14,021 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:15,827 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:34:16,022 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:18,023 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:18,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:34:18,142 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:34:20,024 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:22,025 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:24,026 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:26,027 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:28,028 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:30,028 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:32,029 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:33,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:34:33,416 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:34:34,030 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:36,031 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:38,032 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:40,032 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:42,033 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:44,034 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:45,905 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:34:46,035 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:48,035 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:48,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:34:48,637 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:34:50,036 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:52,037 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:54,038 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:56,039 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:34:58,040 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:00,040 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:02,041 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:03,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:35:03,859 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:35:04,042 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:06,043 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:09,044 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:11,045 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:13,045 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:15,046 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:15,983 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:35:17,047 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:19,048 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:19,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:35:19,147 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:35:21,049 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:23,050 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:25,050 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:27,051 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:29,052 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:31,053 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:33,054 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:34,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:35:34,437 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:35:35,055 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:37,055 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:39,056 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:41,057 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:43,058 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:45,058 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:46,065 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:35:47,059 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:49,060 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:49,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:35:49,720 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:35:51,061 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:53,062 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:55,062 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:57,063 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:35:59,064 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:01,065 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:03,066 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:05,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:36:05,043 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:36:05,066 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:07,067 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:09,068 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:11,069 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:14,070 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:16,071 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:16,144 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:36:18,071 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:20,072 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:20,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:36:20,266 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:36:22,073 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:24,074 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:26,074 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:28,075 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:30,076 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:32,077 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:34,077 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:35,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:36:35,555 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:36:36,078 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:38,079 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:40,080 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:42,081 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:44,081 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:46,082 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:46,223 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:36:48,083 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:50,084 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:50,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:36:50,855 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:36:52,085 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:54,085 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:56,086 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:36:58,087 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:00,088 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:02,089 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:04,089 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:06,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:37:06,081 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:37:06,090 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:08,091 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:10,092 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:12,093 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:14,094 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:16,094 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:16,303 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:37:18,095 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:20,096 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:21,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:37:21,377 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:37:22,098 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:24,098 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:26,099 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:28,100 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:31,101 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:33,102 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:35,102 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:36,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:37:36,603 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:37:37,103 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:39,104 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:41,105 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:43,106 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:45,107 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:46,381 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:37:47,108 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:49,108 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:51,109 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:51,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:37:51,886 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:37:53,110 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:55,111 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:57,112 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:37:59,113 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:01,114 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:03,114 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:05,115 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:07,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:38:07,109 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:38:07,116 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:09,117 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:11,117 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:13,118 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:15,119 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:16,459 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:38:17,119 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:19,120 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:21,121 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:22,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:38:22,425 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:38:23,122 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:25,122 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:27,123 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:29,124 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:31,125 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:33,125 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:35,126 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:37,127 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:37,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:38:37,649 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:38:39,128 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:41,129 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:43,129 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:45,130 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:46,538 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:38:47,131 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:49,131 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:51,132 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:52,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:38:52,956 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:38:53,133 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:55,134 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:57,134 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:38:59,135 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:01,136 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:03,137 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:06,138 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:08,139 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:08,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:39:08,187 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:39:10,140 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:12,140 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:14,141 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:16,142 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:16,617 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:39:18,143 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:20,144 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:22,145 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:23,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:39:23,458 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:39:24,146 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:26,147 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:28,148 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:30,149 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:32,150 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:34,151 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:36,152 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:38,153 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:38,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:39:38,857 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:39:40,154 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:42,155 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:44,156 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:46,157 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:46,697 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:39:48,158 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:50,159 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:52,160 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:54,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:39:54,106 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:39:54,160 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:56,161 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:39:58,162 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:00,163 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:02,164 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:04,165 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:06,166 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:08,167 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:09,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:40:09,423 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:40:10,168 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:12,169 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:14,170 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:16,171 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:16,774 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:40:18,172 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:20,172 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:22,173 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:24,174 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:24,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:40:24,791 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:40:26,175 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:28,175 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:30,176 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:33,178 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:35,179 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:37,180 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:39,180 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:40,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:40:40,040 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:40:41,181 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:43,182 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:45,183 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:46,854 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:40:47,184 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:49,185 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:51,186 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:53,187 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:55,187 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:55,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:40:55,311 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:40:57,188 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:40:59,189 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:01,190 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:03,191 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:05,192 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:07,193 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:09,193 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:10,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:41:10,655 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:41:11,194 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:13,195 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:15,196 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:16,932 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:41:17,197 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:19,197 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:21,198 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:23,199 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:25,199 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:25,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:41:25,880 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:41:27,200 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:29,201 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:31,201 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:33,202 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:35,203 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:37,204 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:39,204 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:41,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:41:41,095 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:41:41,205 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:43,206 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:45,206 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:47,010 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:41:48,207 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:50,208 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:52,209 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:54,210 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:56,210 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:41:57,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:41:57,052 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:41:58,211 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:00,212 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:02,212 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:04,213 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:06,214 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:08,215 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:10,215 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:12,216 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:12,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:42:12,363 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:42:14,217 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:16,218 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:17,087 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:42:18,218 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:20,219 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:22,220 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:24,221 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:26,221 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:27,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:42:27,653 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:42:28,222 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:30,223 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:32,224 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:34,224 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:36,225 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:38,226 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:40,227 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:42,227 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:42,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:42:42,883 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:42:44,228 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:46,229 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:47,165 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:42:48,230 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:50,230 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:52,231 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:54,232 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:56,232 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:42:58,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:42:58,156 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:42:58,233 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:00,234 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:02,235 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:04,235 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:07,236 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:09,237 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:11,238 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:13,239 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:13,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:43:13,424 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:43:15,239 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:17,241 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:17,244 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:43:19,241 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:21,242 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:23,243 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:25,243 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:27,244 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:28,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:43:28,671 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:43:29,245 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:31,245 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:33,246 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:35,247 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:37,248 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:39,248 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:41,249 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:43,250 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:43,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:43:43,895 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:43:45,251 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:47,251 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:47,325 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:43:49,252 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:51,253 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:53,253 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:55,254 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:57,255 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:43:59,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:43:59,120 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:43:59,256 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:01,257 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:03,257 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:05,258 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:07,259 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:09,260 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:11,261 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:13,262 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:14,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:44:14,475 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:44:15,263 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:17,404 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:44:18,264 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:20,265 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:22,265 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:24,266 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:26,267 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:28,268 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:29,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:44:29,698 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:44:30,268 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:32,269 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:34,270 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:36,271 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:38,272 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:40,273 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:42,273 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:44,274 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:44,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:44:44,980 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:44:46,275 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:47,484 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:44:48,276 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:50,277 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:52,277 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:54,278 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:56,279 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:44:58,280 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:00,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:45:00,254 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:45:00,281 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:02,282 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:04,283 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:06,284 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:08,285 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:10,286 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:12,287 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:14,288 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:15,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:45:15,582 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:45:16,289 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:17,564 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:45:18,290 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:20,291 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:22,292 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:24,293 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:26,293 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:28,294 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:30,295 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:30,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:45:30,872 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:45:32,296 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:34,297 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:36,298 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:38,299 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:41,300 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:43,301 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:45,302 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:46,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:45:46,196 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:45:47,303 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:47,644 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:45:49,304 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:51,305 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:53,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:55,307 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:57,308 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:45:59,309 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:01,310 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:01,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:46:01,421 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:46:03,311 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:05,311 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:07,312 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:09,313 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:11,314 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:13,315 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:15,316 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:16,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:46:16,762 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:46:17,317 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:17,726 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:46:19,318 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:21,319 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:23,320 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:25,321 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:27,321 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:29,322 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:31,323 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:32,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:46:32,102 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:46:33,324 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:35,325 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:37,326 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:39,327 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:41,328 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:43,328 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:45,329 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:47,330 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:47,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:46:47,331 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:46:47,806 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:46:49,331 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:51,332 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:53,333 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:55,334 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:57,335 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:46:59,335 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:01,336 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:02,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:47:02,639 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:47:03,337 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:05,338 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:07,338 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:10,339 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:12,340 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:14,341 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:16,341 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:17,885 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:47:17,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:47:17,907 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:47:18,342 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:20,343 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:22,344 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:24,344 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:26,345 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:28,346 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:30,346 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:32,347 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:33,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:47:33,129 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:47:34,348 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:36,349 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:38,350 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:40,351 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:42,351 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:44,352 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:46,353 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:47,964 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:47:48,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:47:48,354 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:47:48,356 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:50,357 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:52,357 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:54,358 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:56,359 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:47:58,359 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:00,360 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:02,361 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:03,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:48:03,630 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:48:04,361 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:06,362 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:09,363 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:11,364 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:13,365 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:15,365 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:17,366 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:18,043 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:48:18,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:48:18,877 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:48:19,367 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:21,368 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:23,368 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:25,369 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:27,370 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:29,371 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:31,371 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:33,372 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:34,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:48:34,627 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:48:35,373 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:37,374 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:39,375 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:41,375 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:43,376 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:45,377 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:47,378 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:48,123 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:48:49,378 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:50,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:48:50,192 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:48:51,379 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:53,380 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:55,381 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:57,382 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:48:59,382 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:01,383 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:04,384 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:05,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:49:05,494 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:49:06,385 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:08,386 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:10,386 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:12,387 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:14,388 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:16,389 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:18,202 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:49:18,389 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:20,390 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:21,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:49:21,113 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:49:22,391 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:24,392 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:26,392 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:28,393 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:30,394 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:32,395 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:34,395 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:36,396 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:36,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:49:36,430 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:49:38,397 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:40,397 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:42,398 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:44,399 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:46,400 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:48,282 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:49:48,401 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:50,401 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:51,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:49:51,744 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:49:52,402 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:54,403 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:56,404 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:49:58,405 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:00,405 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:02,406 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:04,407 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:06,408 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:07,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:50:07,041 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:50:08,409 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:10,409 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:12,410 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:14,411 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:16,412 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:18,361 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:50:18,412 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:20,413 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:22,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:50:22,269 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:50:22,414 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:24,415 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:26,416 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:28,417 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:30,418 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:32,418 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:34,419 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:36,421 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:37,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:50:37,489 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:50:39,422 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:41,422 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:43,423 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:45,424 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:47,425 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:48,441 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:50:49,426 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:51,426 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:52,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:50:52,713 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:50:53,427 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:55,428 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:57,429 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:50:59,429 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:01,430 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:03,431 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:05,432 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:07,432 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:08,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:51:08,008 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:51:09,433 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:11,434 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:13,435 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:15,435 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:17,436 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:18,520 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:51:19,437 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:21,438 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:23,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:51:23,306 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:51:23,439 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:25,439 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:27,440 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:29,441 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:31,441 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:33,442 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:35,443 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:37,444 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:38,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:51:38,605 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:51:39,444 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:41,445 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:43,446 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:45,447 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:47,448 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:48,601 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:51:49,449 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:51,449 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:53,450 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:53,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:51:53,858 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:51:55,451 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:57,452 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:51:59,452 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:01,453 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:03,454 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:05,455 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:07,456 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:09,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:52:09,157 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:52:09,456 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:11,457 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:13,458 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:16,459 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:18,460 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:18,681 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:52:20,461 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:22,462 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:24,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:52:24,381 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:52:24,462 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:26,463 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:28,464 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:30,465 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:32,465 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:34,466 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:36,467 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:38,468 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:39,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:52:39,671 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:52:40,468 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:42,469 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:44,470 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:46,471 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:48,472 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:48,761 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:52:50,472 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:52,473 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:54,474 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:54,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:52:54,993 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:52:56,474 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:52:58,475 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:00,476 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:02,477 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:04,478 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:06,478 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:08,479 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:10,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:53:10,320 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:53:10,480 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:12,481 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:14,482 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:16,483 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:18,484 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:18,841 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:53:20,484 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:22,485 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:24,486 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:25,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:53:25,545 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:53:26,487 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:28,488 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:30,488 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:32,489 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:34,490 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:37,491 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:39,492 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:40,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:53:40,840 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:53:41,493 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:43,494 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:45,494 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:47,495 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:48,921 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:53:49,496 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:51,496 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:53,497 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:55,498 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:56,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:53:56,061 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:53:57,499 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:53:59,499 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:01,500 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:03,501 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:05,502 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:07,502 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:09,503 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:11,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:54:11,346 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:54:11,504 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:13,505 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:15,505 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:17,506 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:19,001 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:54:19,507 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:21,508 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:23,509 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:25,509 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:26,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:54:26,649 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:54:27,510 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:29,511 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:31,511 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:33,512 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:35,513 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:37,514 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:39,515 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:41,515 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:41,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:54:41,864 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:54:43,516 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:45,517 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:48,518 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:49,078 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:54:50,519 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:52,519 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:54,520 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:56,521 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:54:57,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:54:57,248 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:54:58,522 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:00,523 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:02,524 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:04,525 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:06,526 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:08,528 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:10,528 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:12,530 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:12,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:55:12,557 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:55:14,531 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:16,532 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:18,532 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:19,157 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:55:20,533 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:22,534 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:24,535 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:26,536 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:27,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:55:27,778 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:55:28,537 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:30,538 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:32,539 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:34,540 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:36,541 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:38,542 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:40,543 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:42,544 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:43,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:55:43,054 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:55:44,545 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:46,546 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:48,547 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:49,236 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:55:50,548 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:53,549 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:55,550 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:57,551 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:55:58,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:55:58,279 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:55:59,552 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:01,553 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:03,554 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:05,556 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:07,557 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:09,558 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:11,559 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:13,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:56:13,499 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:56:13,560 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:15,561 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:17,562 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:19,315 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:56:19,563 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:21,564 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:23,565 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:25,566 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:27,567 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:28,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:56:28,800 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:56:29,568 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:31,569 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:33,570 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:35,571 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:37,572 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:39,573 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:41,574 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:43,575 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:44,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:56:44,041 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:56:45,576 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:47,577 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:49,394 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:56:49,578 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:51,579 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:53,580 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:55,581 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:57,582 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:56:59,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:56:59,283 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:56:59,583 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:01,584 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:03,585 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:05,586 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:07,587 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:09,588 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:11,589 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:14,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:57:14,507 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:57:14,590 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:16,591 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:18,592 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:19,473 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:57:20,593 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:22,594 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:24,595 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:26,596 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:28,597 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:29,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:57:29,729 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:57:30,598 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:32,600 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:34,601 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:36,602 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:38,603 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:40,604 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:42,605 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:44,606 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:45,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:57:45,058 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:57:46,607 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:48,608 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:49,551 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:57:50,609 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:52,610 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:54,611 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:56,612 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:57:58,613 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:00,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:58:00,291 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:58:00,614 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:02,615 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:04,616 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:06,617 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:08,618 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:10,619 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:12,620 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:14,621 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:15,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:58:15,538 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:58:16,622 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:18,623 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:19,631 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:58:20,624 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:22,625 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:24,626 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:26,627 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:29,628 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:30,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:58:30,864 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:58:31,629 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:33,630 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:35,632 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:37,633 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:39,633 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:41,634 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:43,635 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:45,636 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:46,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:58:46,134 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:58:47,637 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:49,638 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:49,709 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:58:51,640 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:53,641 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:55,642 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:57,643 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:58:59,644 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:01,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:59:01,365 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:59:01,645 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:03,645 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:05,646 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:07,647 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:09,648 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:11,649 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:13,650 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:15,651 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:16,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:59:16,731 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:59:17,652 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:19,653 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:19,786 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:59:21,654 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:23,655 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:25,656 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:27,657 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:29,658 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:31,659 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:31,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:59:31,975 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:59:33,660 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:35,661 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:37,662 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:39,663 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:41,664 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:43,665 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:45,666 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:47,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 17:59:47,253 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 17:59:47,667 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:49,668 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:49,865 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 17:59:51,669 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:53,669 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:55,670 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:57,671 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 17:59:59,672 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:01,673 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:02,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:00:02,625 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:00:03,673 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:05,674 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:07,675 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:09,676 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:11,676 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:13,677 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:15,678 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:17,679 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:18,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:00:18,068 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:00:19,680 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:19,944 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:00:21,681 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:23,682 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:26,683 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:28,684 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:30,685 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:32,685 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:33,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:00:33,394 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:00:34,686 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:36,687 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:38,688 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:40,688 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:42,689 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:44,690 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:46,691 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:48,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:00:48,686 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:00:48,692 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:50,025 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:00:50,692 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:52,693 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:54,694 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:56,695 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:00:58,696 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:00,696 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:02,697 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:03,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:01:03,988 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:01:04,698 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:06,699 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:08,699 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:10,700 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:12,701 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:14,702 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:16,702 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:18,703 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:19,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:01:19,288 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:01:20,104 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:01:20,704 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:22,705 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:24,705 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:26,706 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:28,707 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:30,708 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:32,708 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:34,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:01:34,596 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:01:34,709 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:36,710 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:38,710 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:40,711 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:42,712 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:44,713 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:46,714 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:48,714 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:49,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:01:49,876 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:01:50,183 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:01:50,715 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:52,716 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:54,717 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:56,717 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:01:58,718 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:00,719 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:02,720 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:04,721 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:05,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:02:05,100 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:02:06,721 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:08,722 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:10,723 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:13,724 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:15,725 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:17,726 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:19,726 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:20,264 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:02:20,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:02:20,366 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:02:21,727 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:23,728 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:25,729 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:27,730 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:29,731 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:31,731 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:33,732 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:35,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:02:35,648 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:02:35,733 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:37,734 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:39,735 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:41,735 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:43,736 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:45,737 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:47,737 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:49,738 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:50,343 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:02:50,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:02:50,869 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:02:51,739 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:53,740 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:55,741 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:57,742 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:02:59,742 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:01,743 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:03,744 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:05,745 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:06,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:03:06,153 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:03:07,746 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:09,746 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:11,747 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:13,748 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:15,749 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:17,750 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:19,750 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:20,420 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:03:21,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:03:21,461 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:03:21,751 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:23,752 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:25,753 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:27,754 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:29,754 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:31,755 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:33,756 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:35,757 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:36,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:03:36,744 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:03:37,758 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:39,759 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:41,760 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:43,760 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:45,761 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:47,762 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:49,763 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:50,500 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:03:51,764 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:52,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:03:52,054 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:03:53,765 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:55,765 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:57,766 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:03:59,767 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:01,768 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:03,769 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:06,770 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:07,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:04:07,324 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:04:08,771 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:10,772 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:12,772 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:14,773 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:16,774 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:18,775 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:20,580 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:04:20,775 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:22,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:04:22,629 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:04:22,776 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:24,777 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:26,778 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:28,780 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:30,781 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:32,782 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:34,783 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:36,784 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:37,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:04:37,965 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:04:38,785 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:40,786 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:42,787 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:44,788 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:46,789 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:48,790 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:50,660 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:04:50,791 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:52,792 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:53,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:04:53,189 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:04:54,793 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:56,794 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:04:58,795 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:00,796 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:02,797 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:04,798 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:06,799 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:08,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:05:08,528 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:05:08,800 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:10,801 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:12,802 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:14,803 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:17,804 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:19,805 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:20,740 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:05:21,806 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:23,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:05:23,801 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:05:23,808 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:25,809 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:27,810 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:29,811 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:31,812 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:33,813 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:35,814 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:37,815 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:39,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:05:39,057 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:05:39,816 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:41,817 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:43,818 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:45,819 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:47,820 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:49,821 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:50,820 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:05:51,824 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:53,825 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:54,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:05:54,340 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:05:55,826 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:57,827 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:05:59,828 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:06:01,829 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:06:03,830 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:06:05,831 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:06:07,832 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:06:09,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:06:09,566 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:06:09,833 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:06:20,901 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:06:24,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:06:24,800 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:06:40,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:06:40,048 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:06:50,983 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:06:55,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:06:55,511 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:07:10,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:07:10,758 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:07:21,062 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:07:22,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:07:22,850 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 18:07:22,899 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 18:07:22,974 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 18:07:23,900 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:07:23,900 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 18:07:24,900 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:07:26,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:07:26,005 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:07:29,903 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:07:41,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:07:41,230 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:07:47,911 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:07:51,151 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:07:56,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:07:56,456 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:08:11,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:08:11,687 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:08:21,237 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:08:26,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:08:26,915 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:08:42,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:08:42,146 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:08:51,324 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:08:57,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:08:57,373 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:09:12,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:09:12,716 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:09:21,407 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:09:27,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:09:27,943 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:09:43,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:09:43,167 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:09:51,491 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:09:58,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:09:58,397 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:10:13,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:10:13,632 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:10:21,578 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:10:28,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:10:28,861 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:10:44,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:10:44,086 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:10:51,663 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:10:59,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:10:59,322 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:11:14,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:11:14,550 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:11:21,749 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:11:29,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:11:29,780 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:11:45,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:11:45,012 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:11:51,834 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:11:52,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:52,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:52,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:52,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:52,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:52,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:52,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:52,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:52,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:52,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:52,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:52,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:52,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:52,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:52,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:52,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:53,034 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:11:53,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:53,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:53,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:53,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:53,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:53,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:53,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:53,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:53,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:53,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:53,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:53,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:53,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:53,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:53,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:53,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:54,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:54,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:54,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:54,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:54,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:54,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:54,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:54,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:54,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:54,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:54,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:54,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:54,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:54,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:54,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:54,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,035 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:11:55,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:55,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:57,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:58,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:58,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:58,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:58,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:58,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:58,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:58,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:58,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:58,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:58,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:58,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:58,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:58,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:58,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:58,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:58,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:11:59,037 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:00,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:12:00,239 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:12:00,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:00,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:01,042 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:02,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:02,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:02,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:02,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:02,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:02,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:02,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:02,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:02,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:02,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:02,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:02,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:02,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:02,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:02,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:02,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:03,043 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:03,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:03,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:03,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:03,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:03,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:03,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:03,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:03,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:03,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:03,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:03,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:03,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:03,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:03,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:03,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:03,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:05,043 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:06,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:06,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:06,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:06,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:06,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:06,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:06,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:06,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:06,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:06,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:06,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:06,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:06,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:06,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:06,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:06,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,044 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:07,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:07,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:08,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:09,045 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:11,046 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:11,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:11,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:11,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:11,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:11,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:11,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:11,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:11,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:11,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:11,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:11,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:11,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:11,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:11,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:11,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:11,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:12,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:12,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:12,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:12,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:12,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:12,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:12,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:12,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:12,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:12,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:12,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:12,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:12,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:12,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:12,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:12,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:13,047 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:15,049 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:15,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:12:15,540 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:12:15,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:15,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:15,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:15,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:15,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:15,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:15,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:15,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:15,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:15,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:15,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:15,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:15,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:15,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:15,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:15,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:17,050 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:17,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:17,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:17,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:17,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:17,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:17,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:17,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:17,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:17,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:17,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:17,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:17,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:17,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:17,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:17,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:17,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:19,050 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:20,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:20,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:20,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:20,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:20,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:20,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:20,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:20,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:20,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:20,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:20,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:20,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:20,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:20,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:20,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:20,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:21,052 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:21,910 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:12:22,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:22,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:22,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:22,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:22,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:22,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:22,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:22,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:22,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:22,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:22,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:22,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:22,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:22,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:22,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:22,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:23,052 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:25,053 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:27,054 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:29,055 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:30,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:12:30,941 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:12:31,056 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:33,056 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:35,057 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:37,058 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:39,059 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:43,061 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:46,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:12:46,193 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:12:48,063 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:50,064 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:51,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:51,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:51,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:51,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:51,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:51,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:51,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:51,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:51,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:51,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:51,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:51,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:51,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:51,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:51,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:51,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:51,990 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:12:53,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:53,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:53,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:53,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:53,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:53,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:53,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:53,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:53,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:53,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:53,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:53,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:53,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:53,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:53,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:53,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:54,066 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:58,067 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:12:58,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:58,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:58,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:58,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:58,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:58,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:58,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:58,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:58,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:58,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:58,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:58,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:58,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:58,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:58,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:12:58,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:01,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:13:01,423 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:13:02,069 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:13:04,070 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:13:08,072 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:13:10,073 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:13:12,074 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:13:15,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:15,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:15,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:15,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:15,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:15,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:15,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:15,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:15,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:15,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:15,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:15,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:15,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:15,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:15,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:15,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,075 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:13:16,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,076 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:16,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:13:16,658 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:13:18,076 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:13:22,066 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:13:22,078 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:13:24,079 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:13:24,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:24,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:24,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:24,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:24,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:24,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:24,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:24,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:24,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:24,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:24,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:24,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:24,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:24,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:24,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:24,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:25,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:25,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:25,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:25,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:25,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:25,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:25,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:25,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:25,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:25,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:25,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:25,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:25,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:25,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:25,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:25,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:26,079 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:13:28,080 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:13:28,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:28,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:28,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:28,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:28,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:28,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:28,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:28,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:28,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:28,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:28,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:28,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:28,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:28,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:28,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:28,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:29,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:29,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:29,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:29,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:29,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:29,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:29,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:29,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:29,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:29,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:29,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:29,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:29,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:29,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:29,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:29,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:30,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:30,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:30,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:30,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:30,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:30,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:30,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:30,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:30,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:30,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:30,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:30,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:30,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:30,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:30,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:30,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:31,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:31,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:31,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:31,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:31,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:31,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:31,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:31,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:31,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:31,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:31,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:31,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:31,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:31,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:31,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:31,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:32,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:13:32,070 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:13:32,082 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:13:34,083 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:13:35,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:35,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:36,084 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:13:36,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:36,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:36,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:36,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:36,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:36,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:36,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:36,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,195 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:37,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:38,085 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:13:38,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:38,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:38,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:38,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:38,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:38,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:38,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:38,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:38,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:38,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:38,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:38,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:38,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:38,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:38,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:38,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:39,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,086 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:13:40,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:40,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:41,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:42,087 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:13:43,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:43,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:43,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:43,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:43,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:43,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:43,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:43,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:43,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:43,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:43,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:43,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:43,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:43,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:43,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:43,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,088 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:13:44,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:44,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:45,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:45,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:45,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:45,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:45,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:45,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:45,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:45,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:45,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:45,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:45,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:45,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:45,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:45,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:45,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:45,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:46,089 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:13:46,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:46,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:46,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:46,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:46,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:46,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:46,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:46,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:46,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:46,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:46,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:46,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:46,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:46,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:46,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:46,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:13:47,372 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:13:47,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,909 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:47,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:48,090 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:13:49,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:49,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:49,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:49,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:49,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:49,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:49,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:49,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:49,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:49,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:49,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:49,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:49,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:49,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:49,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:49,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:50,091 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:13:52,091 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:13:52,146 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:13:53,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:53,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,092 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:13:54,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:54,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:55,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:55,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:55,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:55,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:55,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:55,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:55,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:55,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:55,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:55,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:55,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:55,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:55,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:55,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:55,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:55,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:56,093 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:13:56,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:56,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:56,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:56,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:56,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:56,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:56,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:56,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:56,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:56,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:56,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:56,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:56,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:56,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:56,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:56,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:58,094 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:13:58,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:58,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:58,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:58,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:58,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:58,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:58,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:58,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:58,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:58,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:58,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:58,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:58,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:58,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:58,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:58,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:13:59,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:00,095 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:14:01,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:01,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:01,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:01,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:01,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:01,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:01,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:01,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:01,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:01,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:01,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:01,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:01,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:01,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:01,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:01,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:02,096 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:14:02,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:02,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:02,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:02,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:02,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:02,438 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:02,439 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:02,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:02,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:02,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:02,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:02,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:02,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:02,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:02,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:02,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:02,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:14:02,597 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:14:04,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:04,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:04,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:04,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:04,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:04,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:04,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:04,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:04,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:04,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:04,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:04,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:04,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:04,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:04,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:04,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:04,097 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:14:06,097 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:14:06,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:06,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:06,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:06,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:06,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:06,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:06,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:06,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:06,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:06,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:06,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:06,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:06,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:06,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:06,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:06,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:08,098 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:14:08,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:08,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:08,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:08,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:08,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:08,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:08,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:08,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:08,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:08,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:08,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:08,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:08,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:08,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:08,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:08,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:10,099 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:14:12,100 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:14:14,101 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:14:16,102 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:14:17,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:14:17,838 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:14:18,103 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:14:20,103 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:14:22,104 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:14:22,226 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:14:28,106 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:14:32,108 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:14:33,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:14:33,056 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:14:37,110 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:14:39,111 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:14:43,113 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:14:43,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:43,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:43,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:43,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:43,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:43,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:43,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:43,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:43,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:43,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:43,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:43,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:43,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:43,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:43,867 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:43,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:45,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:45,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:45,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:45,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:45,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:45,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:45,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:45,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:45,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:45,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:45,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:45,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:45,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:45,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:45,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:45,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:47,115 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:14:48,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:48,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:48,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:48,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:48,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:48,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:48,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:48,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:48,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:48,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:48,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:48,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:48,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:48,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:48,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:48,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:14:48,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:14:48,290 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:14:49,115 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:14:52,307 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:14:53,117 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:14:55,118 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:14:57,119 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:01,121 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:03,122 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:03,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:15:03,638 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:15:05,123 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:05,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:05,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:05,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:05,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:05,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:05,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:05,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:05,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:05,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:05,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:05,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:05,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:05,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:05,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:05,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:05,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:06,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:06,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:06,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:06,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:06,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:06,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:06,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:06,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:06,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:06,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:06,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:06,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:06,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:06,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:06,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:06,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:07,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:07,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:07,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:07,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:07,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:07,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:07,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:07,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:07,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:07,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:07,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:07,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:07,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:07,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:07,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:07,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:08,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:08,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:08,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:08,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:08,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:08,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:08,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:08,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:08,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:08,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:08,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:08,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:08,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:08,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:08,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:08,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:09,124 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:11,125 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:12,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:12,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:12,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:12,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:12,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:12,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:12,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:12,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:12,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:12,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:12,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:12,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:12,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:12,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:12,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:12,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:13,126 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:14,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:14,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:14,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:14,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:14,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:14,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:14,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:14,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:14,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:14,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:14,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:14,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:14,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:14,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:14,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:14,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:15,127 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:17,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:17,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:17,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:17,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:17,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:17,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:17,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:17,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:17,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:17,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:17,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:17,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:17,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:17,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:17,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:17,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:18,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:18,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:18,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:18,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:18,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:18,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:18,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:18,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:18,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:18,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:18,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:18,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:18,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:18,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:18,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:18,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:18,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:15:18,864 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:15:19,129 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:19,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:19,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:20,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:20,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:20,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:20,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:21,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:21,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:21,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:21,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:21,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:21,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:21,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:21,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:21,130 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:21,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:21,304 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 18:15:21,520 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 18:15:21,589 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 18:15:22,130 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 18:15:22,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:22,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:22,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:22,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:22,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:22,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:22,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:22,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:22,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:22,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:22,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:22,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:22,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:22,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:22,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:22,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:22,398 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:15:22,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:22,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:23,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:23,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:23,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:23,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:23,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:23,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:23,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:23,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:23,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:23,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:23,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:23,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:23,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:23,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:23,131 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:24,131 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:25,132 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:25,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:25,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:25,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:25,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:25,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:25,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:25,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:25,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:25,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:25,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:25,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:25,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:25,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:25,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:25,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:25,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:26,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:27,133 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:28,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:28,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:29,133 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:29,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:29,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:29,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:29,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:29,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:29,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:29,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:29,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:29,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:29,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:29,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:29,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:29,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:29,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:29,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:29,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:30,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:31,134 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:31,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:31,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:31,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:31,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:31,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:31,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:31,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:31,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:31,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:31,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:31,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:31,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:31,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:31,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:31,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:31,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:33,135 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:34,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:15:34,203 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:15:35,136 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:35,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:35,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:36,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:36,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:36,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:36,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:36,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:36,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:36,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:36,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:36,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:36,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:36,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:36,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:36,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:36,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:36,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:36,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:37,137 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:37,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:37,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:37,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:37,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:37,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:37,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:37,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:37,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:37,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:37,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:37,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:37,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:37,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:37,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:37,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:37,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,138 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:39,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:39,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:40,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:40,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:40,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:40,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:40,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:40,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:40,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:40,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:40,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:40,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:40,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:40,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:40,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:40,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:40,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:40,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:41,139 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:42,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:42,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:42,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:42,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:42,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:42,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:42,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:42,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:42,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:42,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:42,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:42,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:42,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:42,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:42,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:42,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,140 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:43,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,343 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,345 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,346 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,347 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:43,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:44,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:44,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:44,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:44,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:44,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:44,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:44,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:44,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:44,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:44,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:44,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:44,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:44,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:44,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:44,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:44,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:45,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:45,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:45,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:45,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:45,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:45,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:45,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:45,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:45,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:45,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:45,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:45,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:45,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:45,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:45,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:45,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:45,140 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:46,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:46,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:46,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:46,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:46,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:46,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:46,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:46,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:46,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:46,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:46,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:46,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:46,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:46,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:46,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:46,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:47,141 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:48,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:48,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,142 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:49,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:15:49,493 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:15:49,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:49,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:51,143 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:51,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:51,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:51,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:51,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:51,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:51,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:51,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:51,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:51,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:51,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:51,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:51,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:51,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:51,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:51,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:51,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:52,475 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:15:52,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:52,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:52,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:52,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:52,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:52,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:52,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:52,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:52,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:52,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:52,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:52,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:52,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:52,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:52,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:52,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:53,144 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:55,145 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:57,146 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:15:57,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:57,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:57,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:57,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:57,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:57,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:57,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:57,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:57,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:57,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:57,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:57,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:57,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:57,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:57,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:57,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:15:59,147 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:16:01,148 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:16:03,149 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:16:04,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:16:04,840 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:16:05,149 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:16:07,150 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:16:09,151 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:16:13,153 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:16:17,155 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:16:20,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:16:20,212 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:16:21,156 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:16:22,553 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:16:25,158 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:16:29,159 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:16:31,160 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:16:35,162 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:16:35,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:16:35,590 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:16:36,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:36,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:36,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:36,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:36,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:36,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:36,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:36,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:36,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:36,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:36,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:36,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:36,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:36,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:36,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:36,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:37,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:37,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:37,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:37,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:37,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:37,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:37,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:37,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:37,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:37,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:37,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:37,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:37,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:37,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:37,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:37,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:40,164 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:16:40,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:40,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:40,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:40,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:40,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:40,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:40,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:40,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:40,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:40,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:40,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:40,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:40,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:40,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:40,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:40,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:42,165 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:16:46,167 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:16:48,167 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:16:50,168 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:16:50,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:16:50,812 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:16:52,631 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:16:54,170 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:16:54,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:54,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:54,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:54,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:54,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:54,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:54,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:54,972 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:54,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:54,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:54,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:54,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:54,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:54,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:54,980 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:54,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:16:56,171 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:16:58,172 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:00,172 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:01,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:01,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:01,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:01,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:01,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:01,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:01,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:01,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:01,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:01,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:01,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:01,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:01,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:01,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:01,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:01,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:02,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:02,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:02,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:02,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:02,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:02,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:02,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:02,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:02,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:02,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:02,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:02,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:02,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:02,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:02,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:02,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:03,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:03,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:03,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:03,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:03,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:03,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:03,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:03,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:03,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:03,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:03,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:03,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:03,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:03,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:03,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:03,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:04,174 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:04,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:04,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:04,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:04,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:04,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:04,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:04,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:04,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:04,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:04,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:04,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:04,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:04,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:04,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:04,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:04,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:06,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:06,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:06,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:06,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:06,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:06,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:06,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:06,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:06,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:06,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:06,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:06,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:06,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:06,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:06,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:06,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:06,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:17:06,033 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:17:06,175 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:07,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:07,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:07,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:07,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:08,176 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:08,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:08,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:08,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:08,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:08,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:08,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:08,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:08,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:10,177 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:10,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:10,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:10,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:10,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:10,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:10,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:10,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:10,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:10,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:10,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:10,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:10,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:10,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:10,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:10,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:10,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:11,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:12,178 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:12,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:12,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:12,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:12,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:12,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:12,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:12,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:12,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:12,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:12,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:12,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:12,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:12,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:12,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:12,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:12,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:14,178 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:14,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:14,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:14,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:14,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:14,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:14,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:14,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:14,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:14,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:14,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:14,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:14,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:14,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:14,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:14,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:14,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:15,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:15,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:15,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:15,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:15,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:15,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:15,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:15,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:15,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:15,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:15,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:15,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:15,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:15,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:15,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:15,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:16,179 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:16,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:16,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:16,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:16,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:16,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:16,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:16,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:16,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:16,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:16,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:16,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:16,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:16,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:16,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:16,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:16,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:17,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:17,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:17,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:17,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:17,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:17,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:17,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:17,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:17,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:17,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:17,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:17,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:17,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:17,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:17,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:17,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:18,180 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:18,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:18,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:18,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:18,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:18,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:18,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:18,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:18,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:18,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:18,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:18,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:18,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:18,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:18,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:18,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:18,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:19,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,181 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:20,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,863 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,864 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:20,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:21,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:17:21,336 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:17:21,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:21,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:21,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:21,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:21,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:21,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:21,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:21,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:21,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:21,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:21,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:21,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:21,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:21,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:21,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:21,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:22,182 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:22,710 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:17:23,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:23,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:23,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:23,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:23,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:23,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:23,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:23,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:23,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:23,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:23,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:23,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:23,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:23,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:23,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:23,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:24,183 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:26,184 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:27,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:27,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:28,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:28,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:28,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:28,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:28,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:28,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:28,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:28,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:28,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:28,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:28,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:28,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:28,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:28,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:28,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:28,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:28,185 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:29,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:29,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:30,185 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:30,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:30,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:30,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:30,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:30,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:30,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:30,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:30,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:30,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:30,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:30,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:30,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:30,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:30,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:30,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:30,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:32,186 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:33,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:33,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:34,187 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:34,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:34,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:34,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:34,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:34,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:34,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:34,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:34,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:34,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:34,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:34,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:34,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:34,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:34,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:34,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:34,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:35,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:36,188 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:36,596 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:17:36,597 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:17:37,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:37,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:38,189 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:38,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:38,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:38,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:38,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:38,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:38,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:38,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:38,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:38,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:38,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:38,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:38,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:38,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:38,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:38,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:38,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,190 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:40,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:40,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:41,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:41,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:41,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:41,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:41,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:41,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:41,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:41,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:41,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:41,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:41,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:41,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:41,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:41,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:41,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:41,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:42,191 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:44,192 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:46,193 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:47,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:47,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:47,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:47,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:47,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:47,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:47,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:47,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:47,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:47,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:47,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:47,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:47,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:47,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:47,071 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:47,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:17:48,193 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:50,194 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:51,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:17:51,855 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:17:52,195 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:17:52,787 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:17:54,196 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:18:00,198 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:18:04,200 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:18:07,073 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:18:07,073 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:18:08,202 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:18:12,204 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:18:16,205 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:18:19,207 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:18:22,434 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:18:22,434 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:18:22,866 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:18:23,208 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:18:27,210 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:18:28,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:28,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:28,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:28,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:28,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:28,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:28,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:28,486 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:28,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:28,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:28,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:28,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:28,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:28,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:28,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:28,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:29,211 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:18:32,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:32,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:32,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:32,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:32,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:32,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:32,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:32,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:32,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:32,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:32,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:32,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:32,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:32,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:32,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:32,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:33,213 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:18:34,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:34,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:34,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:34,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:34,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:34,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:34,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:34,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:34,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:34,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:34,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:34,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:34,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:34,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:34,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:34,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:35,214 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:18:37,215 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:18:37,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:18:37,795 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:18:41,216 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:18:43,217 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:18:45,218 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:18:49,220 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:18:51,221 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:18:52,942 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:18:53,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:18:53,025 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:18:53,222 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:18:54,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:54,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,121 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,122 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,223 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:18:55,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:55,602 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 18:18:55,817 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 18:18:55,890 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 18:18:56,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:56,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:56,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:56,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:56,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:56,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:56,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:56,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:56,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:56,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:56,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:56,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:56,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:56,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:56,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:56,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:56,223 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:18:56,224 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 18:18:57,224 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:18:59,225 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:18:59,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,406 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:18:59,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:01,226 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:19:02,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:02,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:02,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:02,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:02,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:02,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:02,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:02,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:02,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:02,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:02,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:02,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:02,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:02,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:02,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:02,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:03,226 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:19:03,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:03,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:03,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:03,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:03,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:03,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:03,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:03,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:03,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:03,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:03,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:03,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:03,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:03,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:03,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:03,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:04,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:04,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:04,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:04,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:04,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:04,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:04,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:04,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:04,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:04,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:04,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:04,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:04,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:04,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:04,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:04,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,227 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:19:05,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:05,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:07,228 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:19:07,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:07,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:07,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:07,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:07,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:07,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:07,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:07,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:07,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:07,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:07,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:07,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:07,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:07,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:07,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:07,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:08,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:19:08,353 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:19:08,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:08,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:08,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:08,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:08,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:08,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:08,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:08,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:08,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:08,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:08,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:08,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:08,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:08,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:08,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:08,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,229 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:19:09,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:09,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:10,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,230 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:19:11,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,563 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:11,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:12,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:12,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:12,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:12,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:12,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:12,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:12,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:12,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:12,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:12,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:12,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:12,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:12,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:12,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:12,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:12,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:13,231 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:19:14,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:14,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:14,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:14,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:14,402 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:14,403 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:14,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:14,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:14,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:14,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:14,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:14,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:14,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:14,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:14,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:14,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:15,231 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:19:16,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:16,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:16,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:16,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:16,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:16,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:16,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:16,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:16,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:16,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:16,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:16,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:16,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:16,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:16,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:16,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:17,232 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:19:18,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:18,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,233 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:19:19,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,747 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:19,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:20,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:20,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:20,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:20,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:20,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:20,309 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:20,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:20,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:20,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:20,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:20,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:20,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:20,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:20,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:20,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:20,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:21,234 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:19:22,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:22,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:22,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:22,748 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:22,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:22,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:22,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:22,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:22,752 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:22,753 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:22,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:22,754 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:22,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:22,756 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:22,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:22,757 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,060 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:19:23,235 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:19:23,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,385 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:23,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:19:23,576 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:19:24,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,391 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,398 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,399 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:24,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,236 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:19:25,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:25,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:26,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:26,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:26,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:26,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:26,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:26,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:26,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:26,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:26,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:26,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:26,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:26,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:26,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:26,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:26,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:26,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:27,237 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:19:28,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,910 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:28,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:29,238 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:19:29,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:29,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:29,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:29,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:29,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:29,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:29,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:29,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:29,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:29,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:29,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:29,530 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:29,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:29,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:29,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:29,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,761 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,901 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:30,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:31,239 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:19:33,240 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:19:35,240 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:19:35,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:35,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:35,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:35,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:35,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:35,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:35,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:35,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:35,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:35,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:35,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:35,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:35,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:35,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:35,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:35,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:19:37,241 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:19:38,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:19:38,807 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:19:39,242 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:19:41,243 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:19:43,244 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:19:49,246 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:19:53,138 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:19:53,248 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:19:54,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:19:54,093 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:19:55,248 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:19:59,250 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:20:03,252 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:20:07,253 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:20:09,254 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:20:09,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:20:09,324 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:20:13,256 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:20:15,257 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:20:19,259 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:20:21,259 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:20:22,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:22,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:22,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:22,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:22,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:22,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:22,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:22,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:22,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:22,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:22,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:22,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:22,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:22,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:22,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:22,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:23,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:23,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:23,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:23,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:23,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:23,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:23,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:23,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:23,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:23,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:23,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:23,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:23,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:23,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:23,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:23,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:23,218 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:20:23,260 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:20:24,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:20:24,614 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:20:25,261 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:20:29,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:29,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:29,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:29,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:29,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:29,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:29,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:29,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:29,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:29,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:29,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:29,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:29,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:29,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:29,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:29,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:29,263 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:20:31,264 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:20:33,265 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:20:35,266 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:20:39,268 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:20:39,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:20:39,861 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:20:40,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:40,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:40,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:40,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:41,268 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:20:41,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:41,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:41,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:41,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:41,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:41,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:41,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:41,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:43,269 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:20:44,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:44,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:44,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:44,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:44,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:44,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:44,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:44,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:44,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:44,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:44,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:44,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:44,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:44,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:44,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:44,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:45,270 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:20:46,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,342 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,351 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:46,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:47,271 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:20:47,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:47,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:47,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:47,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:47,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:47,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:47,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:47,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:47,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:47,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:47,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:47,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:47,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:47,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:47,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:47,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,806 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:48,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:49,272 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:20:49,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:49,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:49,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:49,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:49,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:49,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:49,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:49,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:49,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:49,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:49,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:49,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:49,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:49,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:49,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:49,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:51,273 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:20:51,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:51,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:51,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:51,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:51,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:51,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:51,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:51,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:51,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:51,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:51,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:51,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:51,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:51,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:51,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:51,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:52,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:52,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:52,693 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:52,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:52,694 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:52,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:52,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:52,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:52,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:52,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:52,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:52,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:52,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:52,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:52,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:52,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:53,274 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:20:53,296 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:20:54,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:54,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:55,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:20:55,086 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:20:55,275 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:20:56,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,341 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:56,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:57,276 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:20:58,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,079 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,127 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:58,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,277 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:20:59,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,903 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,904 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,905 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:20:59,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:00,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:00,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:00,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:00,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:00,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:00,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:00,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:00,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:00,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:00,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:00,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:00,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:00,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:00,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:00,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:00,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:01,278 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:21:03,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:03,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:03,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:03,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:03,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:03,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:03,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:03,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:03,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:03,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:03,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:03,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:03,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:03,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:03,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:03,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,047 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,279 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:21:04,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:04,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:06,280 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:21:06,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:06,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:06,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:06,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:06,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:06,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:06,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:06,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:06,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:06,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:06,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:06,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:06,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:06,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:06,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:06,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:07,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:07,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:07,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:07,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:07,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:07,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:07,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:07,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:07,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:07,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:07,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:07,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:07,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:07,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:07,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:07,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:08,281 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:21:08,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:08,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:08,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:08,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:08,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:08,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:08,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:08,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:08,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:08,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:08,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:08,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:08,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:08,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:08,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:08,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,560 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:09,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:10,282 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:21:10,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:21:10,306 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:21:10,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:10,444 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:10,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:10,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:10,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:10,448 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:10,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:10,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:10,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:10,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:10,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:10,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:10,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:10,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:10,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:10,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,062 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,063 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,673 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:11,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:12,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:12,141 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:12,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:12,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:12,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:12,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:12,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:12,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:12,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:12,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:12,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:12,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:12,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:12,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:12,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:12,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:12,283 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:21:14,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,283 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:21:14,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:14,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,284 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:21:16,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:16,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:17,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:17,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:17,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:17,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:17,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:17,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:17,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:17,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:17,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:17,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:17,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:17,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:17,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:17,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:17,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:17,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:18,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:18,233 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:18,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:18,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:18,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:18,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:18,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:18,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:18,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:18,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:18,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:18,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:18,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:18,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:18,244 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:18,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:18,285 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:21:20,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:20,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:20,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:20,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:20,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:20,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:20,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:20,024 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:20,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:20,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:20,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:20,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:20,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:20,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:20,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:20,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:20,286 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:21:22,286 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:21:22,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:22,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:22,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:22,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:22,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:22,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:22,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:22,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:22,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:22,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:22,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:22,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:22,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:22,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:22,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:22,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:21:23,380 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:21:24,287 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:21:25,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:21:25,747 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:21:26,288 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:21:32,291 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:21:36,292 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:21:40,294 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:21:41,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:21:41,176 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:21:44,296 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:21:48,298 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:21:52,300 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:21:53,460 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:21:54,301 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:21:56,401 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:21:56,402 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:21:58,303 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:00,304 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:02,305 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:06,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:08,307 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:10,308 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:10,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:10,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:10,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:10,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:10,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:10,989 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:10,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:10,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:10,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:10,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:10,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:10,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:10,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:10,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:10,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:11,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:11,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:11,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:11,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:11,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:11,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:11,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:11,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:11,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:11,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:11,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:11,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:11,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:11,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:11,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:11,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:11,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:11,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:22:11,625 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:22:14,310 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:16,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:16,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:16,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:16,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:16,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:16,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:16,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:16,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:16,114 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:16,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:16,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:16,117 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:16,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:16,118 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:16,119 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:16,120 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:16,311 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:18,312 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:20,313 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:22,314 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:23,540 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:22:25,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:25,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:25,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:25,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:25,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:25,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:25,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:25,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:25,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:25,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:25,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:25,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:25,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:25,907 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 18:22:26,120 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 18:22:26,188 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 18:22:26,316 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:26,316 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 18:22:27,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:22:27,014 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:22:27,316 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:28,317 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:30,318 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:32,319 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:34,319 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:35,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,017 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,018 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,019 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,020 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,843 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:35,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:36,320 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:37,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,507 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,869 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:37,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:38,321 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:39,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:39,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:39,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:39,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:39,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:39,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:39,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:39,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:39,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:39,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:39,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:39,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:39,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:39,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:39,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:39,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,322 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:40,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,589 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:40,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:41,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:41,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:41,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:41,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:41,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:41,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:41,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:41,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:41,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:41,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:41,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:41,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:41,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:41,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:41,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:41,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:22:42,238 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:22:42,323 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:42,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,437 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,441 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,448 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:42,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:43,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:43,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:43,508 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:43,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:43,509 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:43,510 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:43,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:43,511 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:43,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:43,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:43,515 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:43,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:43,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:43,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:43,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:43,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,194 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,198 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,200 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,324 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:44,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,448 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:44,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:45,077 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:45,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:45,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:45,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:45,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:45,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:45,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:45,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:45,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:45,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:45,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:45,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:45,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:45,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:45,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:45,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:46,325 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:47,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:47,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:47,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:47,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:47,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:47,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:47,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:47,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:47,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:47,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:47,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:47,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:47,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:47,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:47,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:47,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,326 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:48,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:48,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,876 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,881 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:49,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:50,327 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:51,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,716 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:51,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,328 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:52,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,384 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,387 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,388 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,389 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,390 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,393 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,395 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,538 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:52,551 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:53,662 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:22:53,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:53,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:53,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:53,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:53,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:53,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:53,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:53,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:53,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:53,819 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:53,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:53,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:53,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:53,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:53,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:53,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:54,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:54,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:54,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:54,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:54,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:54,331 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:54,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:54,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:54,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:54,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:54,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:54,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:54,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:54,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:54,338 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:54,339 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:54,340 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,852 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:55,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:56,331 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:57,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:22:57,496 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:22:57,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:57,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:57,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:57,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:57,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:57,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:57,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:57,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:57,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:57,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:57,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:57,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:57,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:57,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:57,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:57,729 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:58,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:58,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:58,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:58,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:58,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:58,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:58,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:58,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:58,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:58,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:58,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:58,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:58,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:58,280 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:58,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:58,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:58,333 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:22:59,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:59,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:59,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:59,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:59,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:59,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:59,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:59,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:59,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:59,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:59,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:59,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:59,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:59,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:59,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:22:59,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,279 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,281 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,334 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:23:00,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,350 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,354 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,783 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:00,786 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:02,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:02,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:02,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:02,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:02,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:02,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:02,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:02,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:02,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:02,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:02,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:02,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:02,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:02,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:02,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:02,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:02,335 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:23:03,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,156 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,190 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,191 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,777 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,778 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,779 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:03,780 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:04,336 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:23:05,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:05,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:05,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:05,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:05,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:05,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:05,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:05,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:05,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:05,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:05,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:05,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:05,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:05,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:05,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:05,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:06,337 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:23:07,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,173 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:07,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:08,338 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:23:09,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:09,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:09,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:09,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:09,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:09,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:09,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:09,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:09,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:09,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:09,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:09,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:09,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:09,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:09,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:09,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:10,338 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:23:12,339 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:23:12,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:23:12,725 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:23:16,341 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:23:20,343 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:23:23,742 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:23:24,344 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:23:28,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:23:28,091 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:23:28,346 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:23:32,348 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:23:36,350 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:23:38,351 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:23:42,353 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:23:43,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:23:43,414 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:23:44,353 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:23:48,355 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:23:50,356 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:23:52,357 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:23:53,822 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:23:56,359 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:23:58,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:58,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:58,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:58,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:58,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:58,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:58,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:58,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:58,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:58,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:58,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:58,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:58,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:58,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:58,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:58,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:23:58,360 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:23:58,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:23:58,773 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:24:00,361 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:01,784 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:01,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:01,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:01,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:01,789 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:01,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:01,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:01,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:01,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:01,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:01,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:01,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:01,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:01,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:01,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:01,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:04,362 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:04,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:04,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:04,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:04,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:04,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:04,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:04,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:04,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:04,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:04,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:04,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:04,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:04,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:04,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:04,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:04,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:06,363 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:09,365 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:11,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:11,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:11,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:11,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:11,366 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:11,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:11,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:11,597 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:11,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:11,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:11,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:11,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:11,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:13,367 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:13,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:24:14,000 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:24:15,368 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:17,369 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:19,370 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:21,371 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:22,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:22,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:22,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:22,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:22,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:22,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:22,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:22,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:22,224 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:22,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:22,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:22,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:22,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:22,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:22,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:22,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:23,372 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:23,900 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:24:25,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:25,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:25,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:25,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:25,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:25,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:25,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:25,223 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:25,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:25,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:25,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:25,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:25,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:25,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:25,229 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:25,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:25,373 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:27,374 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:28,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:28,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:28,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:28,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:28,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:28,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:28,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:28,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:28,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:28,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:28,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:28,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:28,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:28,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:28,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:28,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:29,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:24:29,374 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:24:29,378 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:30,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,585 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,593 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,787 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,788 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:30,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,379 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:31,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,759 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:31,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,380 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:33,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:33,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,301 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:34,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:35,381 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:35,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:35,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:35,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:35,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:35,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:35,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:35,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:35,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:35,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:35,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:35,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:35,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:35,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:35,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:35,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:35,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:36,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:37,382 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:37,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:37,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:37,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:37,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:37,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:37,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:37,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:37,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:37,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:37,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:37,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:37,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:37,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:37,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:37,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:37,472 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,179 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,180 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,942 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:38,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:39,382 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:40,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:40,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:40,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:40,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:40,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:40,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:40,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:40,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:40,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:40,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:40,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:40,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:40,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:40,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:40,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:40,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:41,383 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:41,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:41,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:41,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:41,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:41,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:41,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:41,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:41,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:41,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:41,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:41,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:41,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:41,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:41,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:41,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:41,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,649 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,660 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,669 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,670 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,672 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:42,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,258 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,300 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,303 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,305 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,306 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,384 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:43,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,938 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,939 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,940 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,941 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,946 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,990 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,991 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,993 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:43,994 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:44,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:44,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:44,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:44,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:44,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:44,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:44,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:44,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:44,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:44,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:44,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:44,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:44,534 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:44,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:44,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:44,536 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:44,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:24:44,926 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:24:45,385 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:46,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:46,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:46,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:46,359 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:46,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:46,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:46,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:46,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:46,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:46,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:46,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:46,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:46,367 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:46,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:46,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:46,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,386 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:47,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,623 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:47,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:48,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:48,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:48,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:48,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:48,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:48,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:48,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:48,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:48,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:48,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:48,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:48,460 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:48,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:48,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:48,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:48,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:49,387 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:49,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:49,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:49,959 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:49,960 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:49,961 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:49,962 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:49,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:49,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:49,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:49,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:49,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:49,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:49,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:49,969 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:49,970 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:49,971 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,163 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,164 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,165 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,166 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,168 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,169 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,172 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,735 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:50,740 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,238 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,245 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,388 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:51,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,824 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:51,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,898 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:52,900 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:53,389 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:53,984 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:24:54,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:54,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:54,999 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:55,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:55,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:55,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:55,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:55,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:55,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:55,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:55,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:55,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:55,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:55,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:55,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:55,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:55,390 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:56,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:56,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:56,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:56,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:56,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:56,236 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:56,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:56,237 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:56,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:56,239 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:56,240 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:56,241 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:56,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:56,242 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:56,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:56,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:57,391 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:24:59,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:59,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:59,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:59,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:59,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:59,009 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:59,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:59,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:59,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:59,023 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:59,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:59,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:59,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:59,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:59,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:59,029 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:24:59,392 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:25:00,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:25:00,154 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:25:03,394 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:25:09,396 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:25:13,398 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:25:15,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:25:15,535 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:25:17,400 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:25:18,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:18,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:18,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:18,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:18,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:18,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:18,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:18,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:18,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:18,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:18,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:18,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:18,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:18,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:18,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:18,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:19,401 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:25:23,402 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:25:24,065 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:25:27,404 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:25:29,405 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:25:30,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:25:30,793 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:25:33,406 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:25:35,407 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:25:40,409 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:25:42,410 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:25:46,016 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:25:46,016 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:25:46,412 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:25:48,412 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:25:50,413 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:25:50,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:50,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:50,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:50,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:50,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:50,548 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:50,549 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:50,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:50,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:50,554 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:50,555 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:50,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:50,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:50,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:50,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:50,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:52,414 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:25:54,144 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:25:54,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:54,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:54,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:54,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:54,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:54,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:54,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:54,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:54,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:54,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:54,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:54,503 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:54,504 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:54,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:54,505 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:54,506 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:56,416 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:25:58,417 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:25:59,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:59,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:59,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:25:59,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:00,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:00,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:00,283 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:00,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:00,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:00,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:00,286 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:00,287 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:00,418 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:00,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:00,555 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 18:26:00,781 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 18:26:00,855 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 18:26:01,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:26:01,316 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:26:01,419 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 18:26:02,419 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:04,420 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:06,421 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:06,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:06,568 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:06,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:06,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:06,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:06,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:06,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:06,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:06,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:06,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:06,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:06,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:06,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:06,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:06,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:06,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:08,422 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:10,423 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:12,424 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:13,762 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:13,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:13,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:13,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:13,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:13,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:13,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:13,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:13,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:13,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:13,773 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:13,774 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:13,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:13,775 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:13,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:13,776 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:14,425 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:16,425 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:16,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:26:16,568 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:26:18,426 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:19,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:19,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:19,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:19,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:19,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:19,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:19,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:19,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:19,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:19,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:19,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:19,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:19,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:19,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:19,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:19,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:20,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:20,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:20,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:20,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:20,049 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:20,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:20,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:20,051 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:20,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:20,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:20,055 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:20,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:20,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:20,057 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:20,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:20,058 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:20,427 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:21,594 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:21,595 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:21,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:21,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:21,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:21,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:21,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:21,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:21,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:21,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:21,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:21,605 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:21,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:21,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:21,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:21,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,428 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:22,488 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,489 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,500 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,501 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,502 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,622 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,628 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,687 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:22,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:23,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:23,366 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:23,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:23,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:23,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:23,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:23,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:23,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:23,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:23,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:23,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:23,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:23,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:23,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:23,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:23,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,226 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,230 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,231 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,234 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,235 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:24,238 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:26:24,429 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:25,139 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,140 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,143 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,146 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,148 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,151 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,830 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,832 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,834 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,835 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,837 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,838 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:25,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:26,430 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:26,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:26,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:26,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:26,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:26,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:26,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:26,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:26,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:26,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:26,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:26,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:26,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:26,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:26,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:26,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:26,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,473 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,474 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,476 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,479 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,481 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,611 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,612 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,614 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,616 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,617 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,618 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,619 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:27,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,005 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,006 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,007 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,008 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,010 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,012 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,015 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,149 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,150 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,153 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,154 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,157 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,159 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,160 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,161 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,162 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,430 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:28,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,928 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,934 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,936 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:28,937 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:29,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:29,085 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:29,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:29,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:29,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:29,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:29,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:29,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:29,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:29,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:29,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:29,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:29,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:29,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:29,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:29,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:30,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:30,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:30,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:30,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:30,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:30,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:30,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:30,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:30,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:30,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:30,334 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:30,335 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:30,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:30,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:30,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:30,337 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:30,431 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:31,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:26:31,842 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:26:32,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,368 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,371 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,372 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,432 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:32,907 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,912 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,913 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,916 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:32,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,599 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,602 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,604 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,607 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,608 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,609 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,610 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,638 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,671 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,677 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,683 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,727 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,728 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:33,734 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,331 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,332 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,333 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,433 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:34,446 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,447 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,449 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,450 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,451 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,452 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,454 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,455 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:34,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:35,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:35,553 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:35,556 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:35,557 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:35,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:35,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:35,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:35,562 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:35,564 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:35,565 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:35,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:35,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:35,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:35,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:35,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:35,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,211 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,212 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,218 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,220 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,221 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,434 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:36,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,807 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,808 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,813 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:36,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:37,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:37,453 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:37,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:37,456 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:37,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:37,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:37,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:37,459 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:37,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:37,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:37,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:37,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:37,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:37,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:37,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:37,465 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:37,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:37,992 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:37,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:37,995 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:37,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:37,996 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:37,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:37,998 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:38,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:38,000 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:38,001 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:38,002 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:38,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:38,003 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:38,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:38,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:38,435 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:39,176 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,177 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,184 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,634 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,636 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,637 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,639 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:39,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,436 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:40,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,710 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,711 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,712 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,714 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,792 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,794 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,796 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,803 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:40,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,307 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,308 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,310 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,311 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,312 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,314 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,317 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,640 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,646 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,647 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,648 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,650 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,651 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:41,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:42,436 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:43,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,275 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,276 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,277 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,278 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,695 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,696 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,698 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,700 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,702 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,704 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,705 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:43,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:44,438 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:45,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:45,178 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:45,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:45,181 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:45,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:45,182 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:45,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:45,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:45,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:45,185 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:45,186 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:45,187 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:45,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:45,188 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:45,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:45,189 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:46,439 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:47,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:26:47,068 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:26:48,439 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:49,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:49,793 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:49,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:49,798 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:49,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:49,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:49,801 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:49,802 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:49,809 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:49,810 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:49,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:49,812 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:49,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:49,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:49,815 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:49,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:52,441 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:54,316 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:26:56,443 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:26:57,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:57,699 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:57,706 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:57,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:57,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:57,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:57,707 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:57,708 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:57,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:57,713 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:57,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:57,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:57,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:57,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:57,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:26:57,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:00,444 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:27:02,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:27:02,411 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:27:04,446 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:27:06,447 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:27:10,449 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:27:14,450 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:27:14,674 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:14,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:14,678 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:14,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:14,679 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:14,680 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:14,681 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:14,682 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:14,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:14,686 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:14,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:14,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:14,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:14,690 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:14,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:14,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:16,451 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:27:17,668 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:27:17,669 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:27:20,453 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:27:22,454 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:27:24,395 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:27:26,456 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:27:28,457 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:27:30,458 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:27:32,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:27:32,923 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:27:34,460 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:27:36,461 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:27:38,462 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:27:40,463 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:27:42,464 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:27:43,512 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:43,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:43,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:43,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:43,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:43,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:43,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:43,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:43,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:43,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:43,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:43,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:43,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:43,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:43,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:43,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:45,750 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:45,751 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:45,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:45,758 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:46,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:46,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:46,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:46,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:46,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:46,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:46,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:46,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:46,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:46,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:46,109 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:46,110 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:46,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:46,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:46,112 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:46,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:46,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:46,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:46,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:46,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:46,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:46,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:46,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:46,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:46,465 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:27:48,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:27:48,255 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:27:48,466 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:27:50,467 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:27:52,468 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:27:54,469 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:27:54,475 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:27:56,470 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:27:56,566 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:56,567 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:56,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:56,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:56,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:56,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:56,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:56,573 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:56,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:56,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:56,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:56,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:56,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:56,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:56,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:56,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:27:58,471 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:28:00,472 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:28:02,473 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:28:03,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:28:03,572 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:28:04,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:04,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:04,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:04,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:04,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:04,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:04,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:04,257 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:04,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:04,260 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:04,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:04,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:04,263 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:04,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:04,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:04,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:04,473 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:28:07,475 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:28:07,732 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:07,733 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:07,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:07,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:07,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:07,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:07,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:07,739 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:07,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:07,741 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:07,742 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:07,743 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:07,744 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:07,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:07,745 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:07,746 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:08,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:08,645 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:08,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:08,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:08,652 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:08,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:08,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:08,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:08,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:08,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:08,654 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:08,655 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:08,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:08,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:08,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:08,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:09,192 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:09,193 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:09,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:09,196 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:09,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:09,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:09,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:09,199 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:09,201 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:09,202 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:09,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:09,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:09,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:09,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:09,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:09,206 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:09,476 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:28:10,204 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:10,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:10,207 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:10,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:10,208 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:10,209 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:10,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:10,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:10,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:10,213 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:10,214 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:10,215 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:10,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:10,216 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:10,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:10,217 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:11,477 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:28:11,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:11,836 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:11,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:11,839 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:11,840 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:11,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:11,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:11,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:11,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:11,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:11,845 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:11,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:11,847 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:11,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:11,848 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:11,849 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,369 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,370 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,375 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,379 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,380 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,381 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,407 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,516 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,517 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,518 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,519 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,521 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,522 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,523 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,525 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:12,526 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:13,478 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:28:13,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:13,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:13,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:13,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:13,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:13,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:13,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:13,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:13,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:13,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:13,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:13,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:13,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:13,987 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:13,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:13,988 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,817 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,820 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,821 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,822 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,823 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,825 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,826 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,827 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,828 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:14,829 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,021 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,022 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,028 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,031 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,033 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,422 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,423 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,424 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,425 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,427 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,428 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,429 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,431 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,432 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,478 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:28:15,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,653 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,656 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,657 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,658 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,659 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,661 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,662 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,665 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:15,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,328 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,973 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,974 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,976 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,977 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,979 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,981 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,982 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,983 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,985 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:16,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:17,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:17,126 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:17,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:17,129 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:17,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:17,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:17,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:17,132 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:17,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:17,134 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:17,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:17,136 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:17,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:17,137 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:17,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:17,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:17,479 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:28:18,352 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:18,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:18,355 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:18,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:18,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:18,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:18,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:18,358 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:18,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:18,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:18,361 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:18,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:18,363 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:18,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:18,364 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:18,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:18,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:28:18,815 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:28:19,480 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:28:19,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,621 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,624 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,625 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,626 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,627 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,629 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,630 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,631 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,633 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,760 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,763 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,764 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,765 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,766 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,768 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,769 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,770 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,771 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:19,772 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:20,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:20,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:20,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:20,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:20,289 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:20,290 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:20,291 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:20,292 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:20,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:20,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:20,295 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:20,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:20,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:20,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:20,298 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:20,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,032 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,036 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,041 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,078 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,081 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,083 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,084 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,086 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,087 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,088 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,090 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,099 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,106 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,107 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:21,481 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:28:22,261 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,264 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,266 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,267 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,268 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,269 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,272 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,273 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,274 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,915 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,918 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,920 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,923 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:22,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:23,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:23,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:23,461 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:23,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:23,462 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:23,463 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:23,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:23,464 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:23,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:23,466 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:23,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:23,468 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:23,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:23,469 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:23,470 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:23,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:23,482 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:28:24,034 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,037 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,039 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,040 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,043 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,046 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,092 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,093 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,095 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,096 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,097 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,098 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,100 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,101 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,102 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,103 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,105 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:24,554 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:28:25,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:25,243 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:25,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:25,246 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:25,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:25,247 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:25,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:25,249 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:25,250 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:25,251 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:25,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:25,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:25,253 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:25,254 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:25,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:25,255 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:25,482 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:28:26,315 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:26,316 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:26,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:26,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:26,319 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:26,320 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:26,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:26,321 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:26,322 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:26,323 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:26,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:26,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:26,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:26,326 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:26,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:26,327 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,483 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:28:27,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,487 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,490 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,491 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,493 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,496 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,497 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,498 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,499 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,850 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,851 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,853 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,855 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,858 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,861 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,875 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,877 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,880 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:27,886 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,405 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,408 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,409 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,410 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,411 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,412 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,413 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,414 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,415 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,416 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,417 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,418 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,419 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,484 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:28:29,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,571 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,574 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,576 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,577 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,580 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,581 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,582 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,583 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,922 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,925 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,926 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,927 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,930 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,931 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,932 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,933 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,944 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,945 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,949 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,950 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,954 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,955 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:29,956 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:30,884 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:30,885 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:30,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:30,887 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:30,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:30,888 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:30,889 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:30,890 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:30,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:30,891 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:30,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:30,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:30,894 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:30,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:30,895 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:30,896 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:31,485 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:28:31,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:31,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:31,717 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:31,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:31,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:31,719 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:31,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:31,720 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:31,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:31,722 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:31,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:31,723 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:31,724 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:31,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:31,725 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:31,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:33,486 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:28:34,042 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:28:34,042 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:28:39,489 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:28:42,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:42,854 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:42,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:42,859 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:42,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:42,860 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:42,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:42,862 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:42,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:42,868 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:42,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:42,870 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:42,872 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:42,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:42,873 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:42,874 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:43,491 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:28:47,492 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:28:49,265 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:28:49,266 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:28:49,493 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:28:49,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:49,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:49,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:49,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:49,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:49,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:49,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:49,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:49,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:49,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:49,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:49,543 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:49,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:49,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:49,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:49,546 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:53,495 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:28:54,527 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:54,528 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:54,531 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:54,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:54,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:54,533 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:54,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:54,535 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:54,539 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:54,540 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:54,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:54,542 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:54,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:54,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:54,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:54,545 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:28:54,633 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:28:57,496 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:28:59,497 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:03,499 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:04,052 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:04,053 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:04,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:04,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:04,060 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:04,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:04,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:04,061 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:04,064 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:04,065 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:04,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:04,067 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:04,068 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:04,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:04,069 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:04,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:04,584 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:29:04,585 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:29:05,500 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:09,501 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:11,502 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:13,503 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:17,505 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:19,505 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:19,842 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:29:19,842 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:29:21,506 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:23,507 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:24,713 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:29:27,509 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:29,510 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:31,482 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:31,483 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:31,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:31,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:31,510 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:31,958 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:31,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:31,964 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:31,965 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:31,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:31,966 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:31,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:31,967 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:32,144 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 18:29:32,229 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 18:29:32,441 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 18:29:32,516 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:32,519 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 18:29:33,517 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:33,517 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 18:29:35,113 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:29:35,116 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:29:35,518 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:37,519 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:39,520 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:41,521 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:43,522 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:45,523 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:47,524 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:49,525 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:50,356 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:29:50,356 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:29:51,526 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:53,527 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:54,804 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:29:55,528 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:57,528 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:29:59,530 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:02,531 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:04,532 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:06,533 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:06,590 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:30:06,591 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:30:08,534 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:10,535 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:12,536 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:14,537 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:16,538 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:18,539 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:20,540 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:21,911 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:30:21,911 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:30:22,541 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:24,542 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:24,883 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:30:26,544 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:28,544 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:30,545 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:32,546 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:34,547 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:36,548 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:37,197 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:30:37,197 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:30:38,549 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:40,550 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:42,551 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:44,552 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:46,553 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:48,554 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:50,555 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:52,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:30:52,537 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:30:52,556 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:54,557 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:54,962 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:30:56,558 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:30:58,559 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:00,560 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:02,561 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:04,562 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:06,563 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:07,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:31:07,806 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:31:09,564 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:11,565 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:13,566 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:15,566 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:17,567 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:19,568 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:21,569 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:23,050 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:31:23,050 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:31:23,569 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:25,041 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:31:25,570 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:27,571 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:29,572 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:31,573 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:33,573 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:35,574 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:37,575 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:38,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:31:38,353 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:31:39,578 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:41,579 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:43,580 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:45,581 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:47,582 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:49,582 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:51,583 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:53,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:31:53,579 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:31:53,584 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:55,120 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:31:55,585 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:57,585 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:31:59,586 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:01,587 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:03,588 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:05,588 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:07,589 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:08,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:32:08,900 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:32:09,590 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:11,591 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:14,592 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:16,593 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:18,594 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:20,594 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:22,595 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:24,123 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:32:24,123 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:32:24,596 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:25,199 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:32:26,596 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:28,597 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:30,598 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:32,599 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:34,600 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:36,600 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:38,601 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:39,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:32:39,378 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:32:40,604 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:42,604 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:44,605 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:46,606 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:48,607 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:50,607 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:52,608 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:54,609 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:54,715 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:32:54,716 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:32:55,278 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:32:56,609 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:32:58,610 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:00,611 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:02,612 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:04,612 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:06,613 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:08,614 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:10,014 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:33:10,014 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:33:10,614 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:12,615 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:14,616 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:16,617 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:18,618 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:20,619 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:22,619 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:24,620 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:25,282 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:33:25,283 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:33:25,506 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:33:26,621 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:28,622 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:30,623 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:32,624 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:34,625 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:36,626 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:38,627 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:40,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:33:40,572 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:33:40,628 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:42,628 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:45,629 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:47,630 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:49,631 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:51,632 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:53,633 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:55,436 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:33:55,634 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:55,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:33:55,902 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:33:57,635 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:33:59,636 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:01,637 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:03,638 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:05,639 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:07,640 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:09,641 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:11,138 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:34:11,138 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:34:11,642 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:13,643 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:15,644 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:17,645 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:19,645 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:21,646 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:23,647 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:25,515 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:34:25,648 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:26,426 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:34:26,427 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:34:27,650 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:29,650 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:31,651 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:33,652 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:35,652 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:37,653 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:39,654 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:41,655 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:41,731 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:34:41,731 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:34:43,656 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:45,657 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:47,658 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:49,659 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:51,660 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:53,662 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:55,593 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:34:55,663 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:56,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:34:56,963 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:34:57,664 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:34:59,664 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:01,665 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:03,666 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:05,667 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:07,668 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:09,669 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:11,670 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:12,227 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:35:12,228 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:35:13,671 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:15,672 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:17,673 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:19,674 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:21,675 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:23,676 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:25,677 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:25,681 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:35:27,591 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:35:27,591 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:35:28,678 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:30,679 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:32,680 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:34,682 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:36,683 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:38,684 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:40,685 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:42,686 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:42,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:35:42,878 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:35:44,687 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:46,688 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:48,689 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:50,690 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:52,691 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:54,692 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:55,750 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:35:56,693 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:35:58,152 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:35:58,152 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:35:58,694 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:00,695 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:02,696 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:04,697 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:06,698 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:08,699 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:10,700 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:12,701 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:13,378 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:36:13,378 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:36:14,702 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:16,703 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:18,704 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:20,705 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:22,706 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:24,707 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:25,829 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:36:26,708 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:28,601 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:36:28,602 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:36:28,709 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:30,710 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:32,711 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:34,712 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:36,713 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:38,714 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:40,715 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:42,716 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:43,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:36:43,894 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:36:44,717 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:46,718 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:48,719 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:50,720 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:52,721 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:54,722 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:55,909 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:36:56,723 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:58,724 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:36:59,252 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:36:59,252 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:37:00,725 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:02,726 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:04,727 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:06,728 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:08,729 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:10,729 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:12,731 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:14,477 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:37:14,477 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:37:15,732 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:17,733 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:19,734 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:21,735 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:23,736 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:25,737 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:25,989 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:37:27,738 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:29,709 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:37:29,709 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:37:29,739 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:31,740 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:33,741 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:35,742 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:37,743 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:39,744 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:41,745 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:43,746 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:44,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:37:44,969 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:37:45,747 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:47,748 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:49,749 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:51,750 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:53,751 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:55,752 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:56,069 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:37:57,753 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:37:59,754 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:00,304 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:38:00,305 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:38:01,755 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:03,756 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:05,757 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:07,758 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:09,759 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:11,760 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:13,761 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:15,537 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:38:15,537 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:38:15,762 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:17,763 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:19,764 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:21,765 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:23,766 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:25,767 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:26,149 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:38:27,768 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:29,770 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:30,755 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:38:30,756 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:38:31,770 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:33,771 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:35,772 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:37,773 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:39,774 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:42,775 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:44,776 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:46,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:38:46,125 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:38:46,777 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:48,778 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:50,779 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:52,780 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:54,781 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:56,230 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:38:56,782 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:38:58,783 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:00,784 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:01,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:39:01,435 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:39:02,785 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:04,786 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:06,787 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:08,788 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:10,789 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:12,790 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:14,791 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:16,664 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:39:16,665 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:39:16,792 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:18,793 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:20,794 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:22,795 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:24,796 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:26,310 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:39:26,797 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:28,798 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:30,799 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:31,952 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:39:31,952 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:39:33,800 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:35,801 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:37,802 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:39,803 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:41,804 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:43,805 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:45,806 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:47,344 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:39:47,344 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:39:47,807 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:49,808 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:51,809 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:53,810 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:55,811 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:56,390 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:39:57,812 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:39:59,813 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:01,814 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:02,643 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:40:02,643 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:40:03,815 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:05,816 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:07,817 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:09,818 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:11,819 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:13,820 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:15,821 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:17,822 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:17,883 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:40:17,883 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:40:19,823 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:21,824 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:23,825 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:25,826 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:26,469 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:40:27,827 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:29,828 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:31,829 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:33,104 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:40:33,104 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:40:33,830 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:35,831 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:37,832 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:39,833 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:41,834 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:43,835 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:46,836 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:48,329 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:40:48,329 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:40:48,837 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:50,838 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:52,839 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:54,840 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:56,548 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:40:56,841 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:40:58,842 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:00,844 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:02,845 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:03,575 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:41:03,575 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:41:04,846 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:06,847 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:08,847 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:10,848 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:12,849 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:14,850 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:16,851 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:18,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:41:18,800 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:41:18,852 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:20,853 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:22,854 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:24,855 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:26,627 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:41:26,856 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:28,857 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:30,858 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:32,858 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:34,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:41:34,014 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:41:34,859 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:36,860 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:38,861 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:40,862 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:42,863 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:44,864 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:46,865 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:48,865 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:49,383 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:41:49,384 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:41:50,866 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:52,867 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:54,868 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:56,705 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:41:56,869 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:41:58,870 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:00,870 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:02,871 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:04,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:42:04,799 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:42:04,872 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:06,873 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:08,874 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:10,874 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:12,875 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:14,876 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:16,877 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:18,878 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:20,091 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:42:20,092 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:42:20,879 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:22,880 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:24,880 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:26,784 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:42:26,881 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:28,882 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:30,883 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:32,884 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:34,885 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:35,421 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:42:35,421 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:42:37,886 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:39,887 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:41,888 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:43,889 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:45,889 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:47,890 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:49,891 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:50,736 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:42:50,737 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:42:51,892 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:53,893 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:55,893 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:56,862 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:42:57,894 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:42:59,895 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:01,896 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:03,896 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:05,897 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:06,075 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:43:06,075 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:43:07,898 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:09,899 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:11,900 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:13,900 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:15,901 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:17,902 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:19,903 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:21,313 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:43:21,313 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:43:21,904 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:23,904 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:25,905 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:26,940 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:43:27,906 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:29,907 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:31,908 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:33,909 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:35,910 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:36,550 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:43:36,550 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:43:37,910 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:39,911 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:41,912 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:43,913 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:45,914 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:47,914 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:49,915 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:51,857 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:43:51,857 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:43:51,916 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:53,917 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:55,917 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:57,020 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:43:57,918 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:43:59,919 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:01,920 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:03,921 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:05,922 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:07,128 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:44:07,128 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:44:08,923 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:10,923 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:12,924 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:14,925 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:16,926 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:18,927 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:20,928 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:22,404 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:44:22,404 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:44:22,929 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:24,930 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:26,930 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:27,100 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:44:28,931 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:30,932 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:32,933 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:34,934 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:36,934 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:37,632 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:44:37,632 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:44:38,935 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:40,936 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:42,937 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:44,938 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:46,938 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:48,939 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:50,940 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:52,941 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:52,984 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:44:52,984 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:44:54,942 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:57,180 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:44:57,943 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:44:59,943 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:01,944 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:03,945 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:05,946 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:07,947 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:08,210 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:45:08,210 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:45:09,948 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:11,948 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:13,949 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:15,950 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:17,951 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:19,951 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:21,952 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:23,480 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:45:23,480 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:45:23,953 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:25,954 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:27,259 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:45:27,955 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:29,956 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:31,956 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:33,957 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:35,958 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:37,959 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:38,697 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:45:38,698 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:45:39,960 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:41,961 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:43,961 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:45,962 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:47,963 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:49,964 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:51,965 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:53,966 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:54,133 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:45:54,134 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:45:55,966 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:57,339 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:45:57,967 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:45:59,968 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:01,969 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:03,969 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:05,970 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:07,971 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:09,357 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:46:09,357 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:46:09,972 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:11,972 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:13,973 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:15,974 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:17,975 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:19,975 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:21,976 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:24,676 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:46:24,676 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:46:24,977 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:26,978 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:27,418 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:46:28,979 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:30,980 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:32,980 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:34,981 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:36,982 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:38,983 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:39,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:46:39,924 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:46:40,984 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:42,985 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:44,985 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:46,986 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:48,987 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:50,988 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:52,989 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:54,989 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:55,155 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:46:55,155 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:46:57,497 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:46:57,991 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:46:59,991 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:01,992 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:03,993 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:05,994 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:07,995 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:09,996 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:10,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:47:10,374 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:47:11,996 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:13,997 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:15,998 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:17,999 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:20,000 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:22,001 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:24,001 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:25,663 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:47:25,663 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:47:26,002 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:27,576 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:47:28,003 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:30,004 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:32,005 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:34,006 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:36,007 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:38,008 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:40,009 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:40,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:47:40,882 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:47:42,009 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:44,010 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:46,011 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:48,012 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:50,012 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:52,013 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:54,014 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:56,015 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:47:56,271 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:47:56,271 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:47:57,654 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:47:58,016 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:00,016 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:02,017 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:04,018 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:06,019 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:09,020 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:11,021 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:11,570 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:48:11,570 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:48:13,022 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:15,023 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:17,024 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:19,025 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:21,026 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:23,027 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:25,028 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:26,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:48:26,792 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:48:27,029 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:27,732 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:48:29,030 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:31,030 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:33,031 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:35,032 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:37,033 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:39,033 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:41,034 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:42,089 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:48:42,090 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:48:43,035 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:45,036 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:47,037 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:49,038 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:51,039 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:53,040 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:55,041 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:57,042 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:48:57,373 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:48:57,374 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:48:57,812 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:48:59,043 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:01,043 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:03,044 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:05,045 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:07,046 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:09,047 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:11,048 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:12,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:49:12,593 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:49:13,049 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:15,050 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:17,051 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:19,052 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:21,052 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:23,053 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:26,054 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:27,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:49:27,846 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:49:28,055 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:28,692 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:49:30,056 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:32,057 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:34,058 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:36,059 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:38,060 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:40,061 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:42,061 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:43,703 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:49:43,703 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:49:44,062 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:46,063 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:48,064 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:50,065 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:52,066 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:54,066 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:56,067 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:57,972 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:49:58,068 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:49:58,921 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:49:58,921 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:50:00,069 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:02,070 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:04,071 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:06,072 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:08,072 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:10,073 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:12,074 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:14,075 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:14,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:50:14,203 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:50:16,076 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:18,076 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:20,077 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:22,078 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:24,079 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:26,080 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:28,051 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:50:29,082 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:29,552 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:50:29,552 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:50:31,083 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:33,084 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:35,085 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:37,086 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:39,086 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:41,087 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:43,088 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:44,856 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:50:44,857 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:50:45,089 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:47,090 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:49,091 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:51,091 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:53,093 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:55,093 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:57,094 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:50:58,135 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:50:59,095 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:00,082 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:51:00,083 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:51:01,096 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:03,096 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:05,097 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:07,098 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:09,099 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:11,100 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:13,101 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:15,101 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:15,297 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:51:15,297 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:51:17,102 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:19,103 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:21,104 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:23,105 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:25,106 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:27,107 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:28,214 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:51:29,108 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:30,579 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:51:30,580 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:51:31,109 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:33,110 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:35,111 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:37,112 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:39,113 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:41,113 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:43,114 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:45,115 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:45,866 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:51:45,866 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:51:47,116 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:49,116 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:51,117 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:53,118 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:55,119 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:57,120 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:51:58,292 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:51:59,121 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:01,094 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:52:01,094 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:52:01,121 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:03,122 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:05,123 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:07,124 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:09,125 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:11,125 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:13,126 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:15,127 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:16,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:52:16,362 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:52:17,128 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:19,129 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:21,129 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:24,131 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:26,132 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:28,133 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:28,371 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:52:30,134 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:31,588 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:52:31,589 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:52:32,134 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:34,135 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:36,136 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:38,137 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:40,137 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:42,138 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:44,139 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:46,140 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:46,878 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:52:46,879 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:52:48,141 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:50,142 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:52,142 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:54,143 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:56,144 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:58,145 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:52:58,451 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:53:00,146 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:02,146 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:02,174 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:53:02,175 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:53:04,147 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:06,148 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:08,149 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:10,150 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:12,150 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:14,151 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:16,152 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:17,443 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:53:17,443 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:53:18,153 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:20,154 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:22,155 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:24,156 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:26,157 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:28,157 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:28,530 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:53:30,158 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:32,159 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:32,667 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:53:32,667 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:53:34,160 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:36,161 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:38,162 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:40,162 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:42,163 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:45,164 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:47,165 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:47,963 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:53:47,963 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:53:49,166 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:51,167 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:53,168 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:55,169 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:57,169 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:53:58,610 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:53:59,170 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:01,171 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:03,172 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:03,262 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:54:03,263 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:54:05,173 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:07,173 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:09,174 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:11,175 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:13,176 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:15,177 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:17,178 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:18,485 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:54:18,486 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:54:19,178 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:21,179 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:23,180 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:25,181 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:27,181 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:28,691 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:54:29,182 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:31,183 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:33,184 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:33,879 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:54:33,879 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:54:35,185 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:37,185 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:39,186 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:41,187 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:43,188 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:45,189 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:47,190 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:49,108 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:54:49,108 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:54:49,191 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:51,191 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:53,192 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:55,193 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:57,194 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:54:58,770 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:54:59,195 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:01,196 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:03,197 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:04,458 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:55:04,458 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:55:06,198 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:08,199 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:10,200 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:12,201 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:14,202 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:16,203 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:18,204 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:19,685 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:55:19,685 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:55:20,204 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:22,205 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:24,206 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:26,207 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:28,208 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:28,850 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:55:30,208 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:32,209 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:34,210 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:34,908 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:55:34,908 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:55:36,211 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:38,212 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:40,213 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:42,213 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:44,214 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:46,215 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:48,216 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:50,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:55:50,136 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:55:50,216 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:52,217 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:54,218 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:56,219 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:58,219 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:55:58,930 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:56:00,220 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:02,221 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:04,222 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:05,362 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:56:05,362 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:56:06,223 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:08,224 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:10,225 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:12,226 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:14,226 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:16,227 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:18,228 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:20,229 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:20,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:56:20,587 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:56:22,230 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:25,231 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:27,232 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:29,010 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:56:29,233 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:31,234 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:33,234 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:35,235 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:35,917 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:56:35,917 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:56:37,236 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:39,237 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:41,238 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:43,239 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:45,240 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:47,241 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:49,241 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:51,142 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:56:51,142 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:56:51,242 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:53,243 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:55,244 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:57,245 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:56:59,089 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:56:59,246 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:01,247 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:03,248 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:05,248 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:06,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:57:06,365 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:57:07,249 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:09,250 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:11,251 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:13,252 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:15,253 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:17,254 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:19,254 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:21,255 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:21,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:57:21,643 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:57:23,256 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:25,258 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:27,258 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:29,168 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:57:29,259 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:31,260 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:34,261 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:36,262 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:36,865 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:57:36,865 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:57:38,263 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:40,263 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:42,264 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:44,265 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:46,266 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:48,267 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:50,268 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:52,111 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:57:52,112 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:57:52,268 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:54,269 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:56,270 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:58,271 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:57:59,248 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:58:00,272 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:02,272 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:04,273 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:06,274 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:07,394 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:58:07,395 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:58:08,275 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:10,276 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:12,276 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:14,277 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:16,278 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:18,279 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:20,280 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:22,281 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:22,642 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:58:22,643 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:58:24,281 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:26,282 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:28,283 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:29,327 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:58:30,284 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:32,284 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:34,285 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:36,286 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:37,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:58:37,894 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:58:38,287 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:40,287 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:42,288 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:44,289 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:46,290 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:48,290 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:50,291 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:52,292 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:53,116 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:58:53,117 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:58:54,293 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:56,294 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:58,294 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:58:59,407 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:59:00,295 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:02,296 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:05,297 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:07,298 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:08,532 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:59:08,533 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:59:09,299 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:11,300 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:13,300 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:15,301 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:17,302 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:19,302 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:21,303 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:23,304 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:23,816 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:59:23,816 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:59:25,305 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:27,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:29,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:29,486 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 18:59:31,307 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:33,308 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:35,309 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:37,310 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:39,038 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:59:39,038 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:59:39,310 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:41,311 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:43,312 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:45,313 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:47,314 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:49,314 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:51,315 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:53,316 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:54,324 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 18:59:54,325 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 18:59:55,317 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:57,318 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:59,319 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 18:59:59,565 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:00:02,320 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:04,321 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:06,322 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:08,322 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:09,615 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:00:09,616 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:00:10,323 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:12,324 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:14,325 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:16,326 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:18,327 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:20,327 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:22,328 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:24,329 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:24,902 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:00:24,902 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:00:26,330 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:28,331 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:29,646 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:00:30,332 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:32,332 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:34,333 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:36,334 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:38,335 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:40,125 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:00:40,126 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:00:40,336 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:42,337 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:44,337 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:46,338 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:48,339 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:50,340 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:52,341 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:54,341 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:55,558 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:00:55,558 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:00:56,342 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:58,343 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:00:59,726 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:01:00,344 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:02,345 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:04,346 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:06,346 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:08,347 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:10,348 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:10,799 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:01:10,799 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:01:12,349 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:14,350 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:17,351 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:19,352 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:21,352 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:23,353 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:25,354 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:26,025 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:01:26,025 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:01:27,355 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:29,356 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:29,807 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:01:31,356 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:33,357 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:35,358 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:37,359 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:39,360 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:41,318 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:01:41,318 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:01:41,360 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:43,361 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:45,362 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:47,363 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:49,364 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:51,364 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:53,365 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:55,366 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:56,635 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:01:56,636 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:01:57,367 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:59,368 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:01:59,889 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:02:01,368 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:03,369 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:05,370 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:08,371 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:10,372 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:11,897 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:02:11,897 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:02:12,373 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:14,374 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:16,375 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:18,375 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:20,376 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:22,377 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:24,378 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:26,379 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:27,170 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:02:27,170 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:02:28,380 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:29,972 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:02:30,380 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:32,381 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:34,382 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:36,383 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:38,384 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:40,385 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:42,385 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:42,396 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:02:42,396 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:02:44,386 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:46,387 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:48,388 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:50,389 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:52,390 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:54,390 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:56,391 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:02:57,767 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:02:57,767 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:02:58,392 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:00,052 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:03:00,393 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:02,394 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:04,395 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:06,396 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:08,396 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:10,397 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:12,398 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:13,045 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:03:13,045 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:03:14,399 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:16,400 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:18,400 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:20,401 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:22,402 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:24,403 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:26,404 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:28,270 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:03:28,270 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:03:28,405 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:30,133 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:03:30,406 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:32,406 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:34,407 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:36,408 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:38,409 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:40,410 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:42,411 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:43,495 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:03:43,495 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:03:44,412 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:46,412 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:48,413 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:51,414 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:53,415 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:55,416 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:57,417 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:03:58,721 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:03:58,722 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:03:59,418 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:00,213 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:04:01,418 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:03,419 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:05,420 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:07,421 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:09,422 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:11,423 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:13,424 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:14,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:04:14,057 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:04:15,424 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:17,425 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:19,426 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:21,427 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:23,428 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:25,429 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:27,430 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:29,360 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:04:29,360 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:04:29,431 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:30,293 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:04:31,432 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:33,433 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:35,434 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:37,435 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:39,436 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:41,437 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:43,438 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:44,586 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:04:44,587 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:04:45,439 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:47,440 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:49,441 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:51,442 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:53,443 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:55,444 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:57,444 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:59,445 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:04:59,899 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:04:59,900 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:05:00,373 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:05:01,446 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:03,447 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:05,448 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:07,449 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:09,450 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:11,451 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:13,452 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:15,124 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:05:15,124 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:05:15,453 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:18,454 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:20,455 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:22,456 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:24,457 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:26,458 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:28,459 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:30,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:05:30,383 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:05:30,460 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:30,627 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:05:32,461 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:34,462 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:36,463 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:38,464 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:40,465 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:42,466 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:44,467 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:45,781 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:05:45,781 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:05:46,468 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:48,469 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:50,470 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:52,471 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:54,472 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:56,473 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:05:58,474 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:00,475 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:00,534 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:06:01,013 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:06:01,013 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:06:02,476 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:04,477 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:06,478 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:08,479 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:10,480 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:12,481 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:14,482 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:16,374 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:06:16,375 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:06:16,482 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:18,483 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:20,485 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:22,486 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:24,486 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:26,487 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:28,488 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:30,489 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:30,613 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:06:31,603 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:06:31,603 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:06:32,490 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:34,491 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:36,492 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:38,493 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:40,494 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:42,495 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:44,496 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:46,497 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:46,882 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:06:46,883 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:06:48,498 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:50,499 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:53,500 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:55,501 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:57,502 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:06:59,503 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:00,692 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:07:01,504 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:02,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:07:02,171 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:07:03,505 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:05,506 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:07,507 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:09,508 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:11,509 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:13,510 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:15,511 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:17,492 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:07:17,492 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:07:17,512 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:19,513 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:21,514 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:23,515 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:25,516 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:27,517 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:29,518 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:30,772 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:07:31,519 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:32,790 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:07:32,790 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:07:33,520 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:35,521 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:37,522 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:39,523 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:41,523 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:43,524 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:45,525 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:47,526 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:48,030 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:07:48,030 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:07:49,528 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:51,529 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:53,530 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:55,530 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:57,531 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:07:59,532 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:00,850 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:08:01,533 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:03,325 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:08:03,326 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:08:03,534 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:05,535 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:07,536 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:09,537 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:11,538 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:13,539 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:15,540 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:17,541 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:18,620 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:08:18,620 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:08:19,542 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:21,543 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:23,544 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:25,545 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:27,546 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:29,547 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:30,930 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:08:31,548 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:33,549 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:33,841 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:08:33,841 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:08:35,550 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:38,551 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:40,552 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:42,553 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:44,554 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:46,555 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:48,556 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:49,175 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:08:49,175 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:08:50,557 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:52,558 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:54,559 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:56,560 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:08:58,562 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:00,563 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:01,010 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:09:02,564 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:04,430 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:09:04,430 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:09:04,564 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:06,565 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:08,566 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:10,567 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:12,568 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:14,569 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:16,570 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:18,571 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:19,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:09:19,726 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:09:20,572 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:22,573 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:24,574 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:26,575 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:28,576 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:30,578 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:31,089 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:09:32,578 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:34,579 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:34,947 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:09:34,947 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:09:36,580 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:38,581 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:40,582 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:42,583 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:44,584 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:46,585 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:48,586 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:50,232 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:09:50,232 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:09:50,587 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:52,588 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:54,589 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:56,590 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:09:58,591 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:00,592 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:01,169 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:10:02,593 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:04,594 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:05,478 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:10:05,478 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:10:06,595 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:08,596 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:10,597 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:12,598 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:14,599 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:16,600 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:18,601 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:20,818 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:10:20,819 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:10:21,602 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:23,603 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:25,605 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:27,606 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:29,606 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:31,248 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:10:31,607 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:33,608 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:35,609 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:36,072 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:10:36,072 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:10:37,610 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:39,612 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:41,612 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:43,613 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:45,614 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:47,615 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:49,616 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:51,293 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:10:51,294 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:10:51,617 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:53,618 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:55,619 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:57,620 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:10:59,622 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:01,327 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:11:01,622 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:03,623 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:05,624 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:06,514 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:11:06,514 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:11:07,625 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:09,626 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:11,627 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:13,628 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:15,629 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:17,630 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:19,631 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:21,632 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:21,893 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:11:21,893 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:11:23,633 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:25,634 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:27,635 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:29,636 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:31,406 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:11:31,637 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:33,638 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:35,639 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:37,131 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:11:37,132 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:11:37,640 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:39,640 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:42,642 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:44,643 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:46,644 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:48,645 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:50,646 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:52,467 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:11:52,468 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:11:52,647 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:54,648 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:56,649 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:11:58,650 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:00,651 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:01,486 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:12:02,652 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:04,653 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:06,654 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:07,691 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:12:07,691 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:12:08,655 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:10,656 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:12,657 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:14,658 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:16,659 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:18,660 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:20,661 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:22,662 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:22,919 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:12:22,919 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:12:24,663 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:26,664 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:28,665 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:30,666 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:31,565 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:12:32,667 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:34,668 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:36,669 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:38,135 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:12:38,136 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:12:38,670 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:40,671 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:42,672 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:44,673 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:46,674 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:48,675 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:50,676 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:52,676 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:53,392 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:12:53,392 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:12:54,677 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:56,678 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:12:58,679 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:00,680 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:01,645 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:13:02,681 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:04,682 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:06,683 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:08,738 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:13:08,738 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:13:09,684 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:11,685 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:13,686 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:15,687 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:17,688 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:19,689 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:21,690 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:23,691 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:24,035 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:13:24,035 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:13:25,692 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:27,693 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:29,694 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:31,695 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:31,724 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:13:33,696 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:35,697 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:37,698 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:39,348 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:13:39,348 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:13:39,699 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:41,700 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:43,701 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:45,702 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:47,703 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:49,704 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:51,705 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:53,706 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:54,644 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:13:54,644 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:13:55,707 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:57,708 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:13:59,709 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:01,710 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:01,803 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:14:03,711 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:05,712 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:07,713 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:09,714 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:09,943 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:14:09,943 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:14:11,715 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:13,715 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:15,716 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:17,717 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:19,717 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:21,718 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:23,720 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:25,167 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:14:25,167 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:14:25,720 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:27,721 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:29,722 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:31,723 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:31,882 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:14:33,724 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:35,725 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:37,726 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:39,727 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:40,436 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:14:40,436 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:14:41,728 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:43,729 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:45,730 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:47,731 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:49,732 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:51,733 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:53,734 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:55,688 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:14:55,688 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:14:55,735 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:14:58,736 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:00,737 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:01,961 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:15:02,739 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:04,740 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:06,741 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:08,742 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:10,743 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:10,948 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:15:10,948 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:15:12,744 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:14,745 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:16,746 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:18,747 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:20,748 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:22,749 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:24,750 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:26,171 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:15:26,172 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:15:26,751 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:28,752 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:30,753 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:32,039 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:15:32,754 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:34,755 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:36,756 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:38,757 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:40,758 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:41,420 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:15:41,420 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:15:42,759 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:44,760 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:46,761 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:48,762 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:50,763 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:52,764 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:54,765 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:56,641 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:15:56,641 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:15:56,766 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:15:58,767 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:00,768 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:02,119 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:16:02,769 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:04,770 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:06,771 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:08,772 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:10,773 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:11,986 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:16:11,986 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:16:12,774 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:14,775 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:16,776 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:18,777 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:20,778 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:22,779 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:24,780 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:26,780 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:27,256 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:16:27,256 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:16:28,781 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:30,782 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:32,197 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:16:32,783 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:34,784 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:36,785 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:38,785 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:40,786 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:42,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:16:42,559 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:16:42,787 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:44,788 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:46,789 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:48,789 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:50,790 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:52,791 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:54,792 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:57,793 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:16:57,831 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:16:57,831 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:16:59,794 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:01,795 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:02,277 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:17:03,796 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:05,797 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:07,798 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:09,798 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:11,799 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:13,056 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:17:13,056 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:17:13,800 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:15,801 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:17,802 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:19,803 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:21,804 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:23,804 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:25,805 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:27,806 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:28,494 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:17:28,494 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:17:29,807 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:31,808 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:32,357 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:17:33,809 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:35,809 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:37,810 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:39,811 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:41,812 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:43,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:17:43,737 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:17:43,813 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:45,813 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:47,814 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:49,815 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:51,816 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:53,817 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:55,817 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:57,818 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:17:58,953 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:17:58,953 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:17:59,819 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:01,820 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:02,438 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:18:03,821 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:05,822 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:07,822 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:09,823 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:11,824 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:13,825 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:14,299 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:18:14,299 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:18:15,825 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:17,826 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:19,827 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:21,828 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:23,828 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:26,829 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:28,830 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:29,524 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:18:29,524 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:18:30,831 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:32,517 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:18:32,832 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:34,833 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:36,834 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:38,835 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:40,836 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:42,837 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:44,805 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:18:44,806 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:18:44,837 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:46,839 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:48,840 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:50,841 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:52,841 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:54,842 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:56,843 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:18:58,844 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:00,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:19:00,028 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:19:00,845 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:02,595 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:19:02,846 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:04,847 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:06,848 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:08,849 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:10,850 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:12,851 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:14,852 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:15,386 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:19:15,386 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:19:16,852 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:18,853 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:20,854 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:22,855 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:24,856 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:26,857 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:28,858 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:30,684 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:19:30,685 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:19:30,858 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:32,673 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:19:32,859 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:34,860 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:36,861 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:38,862 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:40,863 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:42,864 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:44,865 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:45,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:19:45,907 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:19:46,866 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:49,867 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:51,868 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:53,869 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:55,870 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:57,871 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:19:59,872 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:01,205 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:20:01,205 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:20:01,873 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:02,753 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:20:03,874 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:05,875 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:07,876 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:09,876 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:11,877 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:13,878 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:15,879 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:16,435 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:20:16,435 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:20:17,880 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:19,881 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:21,882 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:23,883 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:25,884 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:27,884 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:29,885 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:31,718 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:20:31,718 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:20:31,886 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:32,831 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:20:33,887 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:35,888 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:37,888 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:39,889 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:41,890 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:43,891 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:45,892 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:47,222 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:20:47,222 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:20:47,893 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:49,893 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:51,894 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:53,895 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:55,896 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:57,896 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:20:59,898 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:01,898 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:02,445 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:21:02,446 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:21:02,910 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:21:03,899 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:05,900 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:07,901 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:09,901 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:11,902 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:13,903 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:15,904 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:17,675 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:21:17,675 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:21:17,904 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:19,905 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:21,906 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:23,907 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:25,908 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:27,908 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:29,909 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:32,892 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:21:32,892 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:21:32,911 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:33,121 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:21:34,912 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:36,912 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:38,913 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:40,914 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:42,915 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:44,916 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:46,917 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:48,203 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:21:48,203 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:21:48,918 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:50,919 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:52,920 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:54,920 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:56,921 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:21:58,922 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:00,923 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:02,924 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:03,069 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:22:03,559 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:22:03,560 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:22:04,925 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:06,926 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:08,927 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:10,928 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:12,929 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:14,930 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:16,931 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:18,804 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:22:18,805 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:22:18,932 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:20,933 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:22,934 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:24,935 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:26,936 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:28,937 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:31,939 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:33,149 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:22:33,940 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:34,027 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:22:34,027 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:22:35,941 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:37,942 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:39,943 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:41,944 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:43,945 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:45,946 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:47,947 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:49,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:22:49,349 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:22:49,948 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:51,949 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:53,950 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:55,951 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:57,952 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:22:59,953 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:01,954 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:03,228 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:23:03,955 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:04,572 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:23:04,572 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:23:05,956 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:07,957 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:09,958 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:11,959 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:13,960 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:15,961 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:17,962 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:19,924 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:23:19,924 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:23:19,963 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:22,965 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:24,966 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:26,967 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:28,968 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:30,969 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:32,970 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:33,307 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:23:34,971 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:35,296 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:23:35,297 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:23:36,972 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:38,973 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:40,974 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:42,975 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:44,976 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:46,977 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:48,978 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:50,529 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:23:50,529 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:23:50,979 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:52,980 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:54,981 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:56,982 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:23:58,983 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:00,984 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:02,985 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:03,386 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:24:04,986 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:05,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:24:05,750 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:24:06,987 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:08,988 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:10,989 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:12,990 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:14,991 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:16,992 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:18,993 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:20,968 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:24:20,969 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:24:20,994 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:22,995 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:24,996 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:26,997 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:28,998 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:30,999 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:33,000 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:33,465 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:24:35,001 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:36,330 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:24:36,330 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:24:37,001 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:39,002 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:41,003 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:43,004 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:46,005 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:48,006 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:50,007 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:51,587 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:24:51,587 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:24:52,007 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:54,008 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:56,009 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:24:58,010 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:00,011 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:02,012 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:03,544 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:25:04,013 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:06,013 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:06,811 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:25:06,811 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:25:08,014 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:10,015 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:12,016 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:14,017 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:16,018 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:18,019 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:20,020 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:22,021 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:22,219 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:25:22,219 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:25:24,022 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:26,023 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:28,024 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:30,025 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:32,026 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:33,623 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:25:34,026 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:36,027 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:37,442 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:25:37,442 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:25:38,028 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:40,029 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:42,030 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:44,031 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:46,032 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:48,033 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:50,034 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:52,035 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:52,666 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:25:52,667 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:25:54,036 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:56,036 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:25:58,037 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:00,038 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:02,039 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:03,701 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:26:04,040 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:06,041 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:07,906 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:26:07,906 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:26:08,042 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:10,043 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:12,044 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:15,045 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:17,046 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:19,047 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:21,047 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:23,048 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:23,145 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:26:23,145 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:26:25,049 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:27,050 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:29,051 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:31,052 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:33,053 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:33,780 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:26:35,054 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:37,054 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:38,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:26:38,952 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:26:39,055 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:41,056 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:43,057 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:45,058 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:47,059 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:49,059 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:51,060 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:53,061 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:54,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:26:54,259 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:26:55,062 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:57,062 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:26:59,063 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:01,064 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:03,065 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:03,860 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:27:05,066 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:07,067 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:09,067 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:09,484 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:27:09,485 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:27:11,068 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:13,069 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:15,070 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:17,071 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:19,072 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:21,072 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:23,073 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:24,846 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:27:24,846 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:27:25,074 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:27,075 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:29,075 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:31,076 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:33,077 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:33,940 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:27:35,078 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:37,079 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:39,080 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:40,183 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:27:40,184 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:27:41,080 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:43,081 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:45,082 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:47,083 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:49,084 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:52,085 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:54,086 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:55,471 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:27:55,471 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:27:56,087 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:27:58,087 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:00,088 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:02,089 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:04,018 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:28:04,090 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:06,091 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:08,091 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:10,092 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:10,797 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:28:10,797 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:28:12,093 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:14,094 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:16,095 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:18,095 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:20,096 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:22,097 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:24,098 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:26,048 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:28:26,048 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:28:26,099 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:28,100 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:30,101 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:32,101 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:34,099 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:28:34,102 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:36,103 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:38,104 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:40,105 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:41,397 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:28:41,398 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:28:42,105 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:44,106 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:46,107 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:48,108 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:50,108 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:52,109 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:54,110 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:56,111 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:28:56,701 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:28:56,702 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:28:58,112 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:00,112 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:02,113 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:04,114 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:04,179 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:29:06,115 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:08,116 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:10,116 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:11,935 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:29:11,935 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:29:12,117 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:14,118 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:16,119 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:18,120 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:20,120 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:22,121 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:24,122 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:27,123 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:27,158 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:29:27,158 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:29:29,124 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:31,125 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:33,126 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:34,259 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:29:35,127 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:37,128 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:39,129 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:41,130 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:42,547 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:29:42,547 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:29:43,131 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:45,132 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:47,133 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:49,134 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:51,134 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:53,135 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:55,136 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:57,137 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:29:57,782 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:29:57,782 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:29:59,138 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:01,139 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:03,140 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:04,339 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:30:05,140 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:07,141 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:09,142 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:11,143 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:13,004 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:30:13,005 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:30:13,144 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:15,145 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:17,146 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:19,146 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:21,147 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:23,148 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:25,149 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:27,149 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:28,365 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:30:28,365 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:30:29,150 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:31,151 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:33,152 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:34,418 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:30:35,153 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:37,153 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:39,154 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:41,155 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:43,156 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:43,730 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:30:43,730 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:30:45,157 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:47,157 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:49,158 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:51,159 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:53,160 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:55,161 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:57,161 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:30:58,997 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:30:58,998 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:31:00,163 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:02,163 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:04,164 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:04,498 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:31:06,165 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:08,166 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:10,166 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:12,167 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:14,168 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:14,284 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:31:14,285 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:31:16,169 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:18,169 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:20,170 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:22,171 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:24,172 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:26,172 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:28,173 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:29,569 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:31:29,569 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:31:30,174 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:32,174 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:34,175 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:34,577 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:31:36,176 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:38,177 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:40,177 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:42,178 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:44,179 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:44,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:31:44,796 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:31:46,180 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:48,180 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:50,181 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:52,182 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:54,183 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:56,184 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:31:58,185 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:00,115 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:32:00,115 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:32:00,185 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:02,186 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:04,187 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:04,657 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:32:06,188 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:08,188 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:10,189 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:12,190 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:14,190 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:15,382 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:32:15,383 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:32:16,191 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:18,192 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:20,193 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:22,193 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:24,194 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:26,195 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:28,196 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:30,196 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:30,613 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:32:30,613 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:32:32,197 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:34,739 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:32:35,198 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:37,199 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:39,200 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:41,201 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:43,202 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:45,203 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:45,975 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:32:45,975 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:32:47,203 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:49,204 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:51,205 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:53,206 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:55,207 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:57,207 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:32:59,208 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:01,209 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:01,288 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:33:01,288 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:33:03,210 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:04,817 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:33:05,210 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:07,211 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:09,212 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:11,212 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:13,213 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:15,214 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:16,544 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:33:16,544 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:33:17,214 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:19,215 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:21,216 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:23,217 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:25,217 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:27,218 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:29,219 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:31,220 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:31,800 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:33:31,801 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:33:33,221 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:34,898 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:33:35,222 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:37,222 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:39,223 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:41,224 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:44,225 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:46,226 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:47,026 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:33:47,026 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:33:48,226 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:50,227 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:52,228 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:54,229 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:56,230 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:33:58,230 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:00,231 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:02,232 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:02,302 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:34:02,302 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:34:04,233 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:04,980 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:34:06,234 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:08,235 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:10,235 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:12,236 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:14,237 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:16,238 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:17,561 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:34:17,561 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:34:18,239 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:20,239 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:22,240 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:24,241 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:26,242 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:28,243 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:30,243 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:32,244 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:32,785 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:34:32,786 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:34:34,245 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:35,059 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:34:36,246 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:38,247 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:40,247 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:43,249 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:45,249 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:47,250 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:48,130 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:34:48,130 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:34:49,251 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:51,252 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:53,253 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:55,254 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:57,254 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:34:59,255 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:01,256 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:03,257 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:03,433 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:35:03,434 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:35:05,138 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:35:05,258 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:07,259 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:09,260 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:11,261 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:13,262 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:15,263 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:17,264 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:18,726 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:35:18,727 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:35:19,265 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:21,266 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:23,267 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:25,268 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:27,269 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:29,270 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:31,271 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:33,272 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:33,951 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:35:33,951 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:35:35,217 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:35:35,273 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:37,274 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:39,275 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:41,276 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:43,277 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:45,278 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:47,279 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:49,228 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:35:49,229 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:35:49,279 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:51,280 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:53,281 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:55,282 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:57,283 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:35:59,284 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:01,285 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:03,285 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:04,541 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:36:04,541 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:36:05,286 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:05,298 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:36:08,287 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:10,288 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:12,289 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:14,289 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:16,290 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:18,291 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:19,791 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:36:19,791 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:36:20,292 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:22,293 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:24,293 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:26,294 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:28,295 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:30,295 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:32,296 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:34,297 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:35,054 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:36:35,054 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:36:35,378 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:36:36,298 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:38,298 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:40,299 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:42,300 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:44,300 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:46,301 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:48,302 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:50,303 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:50,353 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:36:50,354 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:36:52,303 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:54,304 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:56,305 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:36:58,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:00,306 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:03,307 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:05,308 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:05,459 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:37:05,578 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:37:05,578 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:37:07,309 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:09,310 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:11,310 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:13,311 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:15,312 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:17,313 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:19,313 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:20,957 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:37:20,958 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:37:21,314 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:23,315 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:25,316 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:27,316 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:29,317 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:31,318 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:33,319 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:35,319 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:35,538 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:37:36,248 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:37:36,248 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:37:37,320 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:39,321 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:41,322 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:43,322 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:45,323 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:47,324 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:49,325 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:51,325 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:51,475 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:37:51,476 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:37:53,326 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:55,327 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:57,327 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:37:59,328 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:01,329 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:03,330 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:05,330 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:05,618 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:38:06,795 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:38:06,796 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:38:07,331 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:09,332 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:11,332 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:13,333 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:15,334 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:17,335 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:19,335 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:21,336 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:22,011 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:38:22,012 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:38:23,337 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:25,338 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:27,338 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:29,339 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:31,340 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:34,341 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:35,698 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:38:36,342 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:37,376 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:38:37,376 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:38:38,342 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:40,343 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:42,344 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:44,345 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:46,345 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:48,346 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:50,347 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:52,347 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:52,600 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:38:52,601 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:38:54,348 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:56,349 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:38:58,350 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:00,350 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:02,351 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:04,352 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:05,779 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:39:06,353 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:07,929 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:39:07,930 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:39:08,353 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:10,354 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:12,355 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:14,356 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:16,356 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:18,357 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:20,358 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:22,359 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:23,259 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:39:23,260 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:39:24,360 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:26,360 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:28,361 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:30,362 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:32,363 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:34,363 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:35,859 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:39:36,364 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:38,365 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:38,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:39:38,513 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:39:40,365 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:42,366 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:45,367 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:47,368 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:49,369 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:51,369 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:53,370 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:53,737 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:39:53,738 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:39:55,371 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:57,372 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:39:59,372 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:40:01,373 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:40:03,374 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:40:05,374 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:40:05,939 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:40:07,375 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:40:09,044 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:40:09,044 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:40:09,376 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:40:11,377 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:40:13,377 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:40:15,378 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:40:17,379 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:40:19,379 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:40:21,380 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:40:23,381 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:40:24,349 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:40:24,349 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:40:25,382 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:40:27,383 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:40:30,384 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:40:32,384 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:40:34,385 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:40:36,020 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:40:36,386 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:40:39,592 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:40:39,592 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:40:54,814 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:40:54,815 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:41:06,099 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:41:10,066 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:41:10,066 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:41:25,336 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:41:25,337 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:41:36,178 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:41:40,598 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:41:40,598 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:41:50,440 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: partial_history -2022-10-25 19:41:50,446 DEBUG SenderThread:2194005 [sender.py:send():232] send: history -2022-10-25 19:41:50,496 DEBUG SenderThread:2194005 [sender.py:send():232] send: summary -2022-10-25 19:41:50,595 INFO SenderThread:2194005 [sender.py:_save_file():946] saving file wandb-summary.json with policy end -2022-10-25 19:41:51,414 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/wandb-summary.json -2022-10-25 19:41:52,415 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:41:55,844 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:41:55,846 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:41:58,418 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:42:06,289 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:42:11,070 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:42:11,070 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:42:14,425 INFO Thread-7 :2194005 [dir_watcher.py:_on_file_modified():226] file/dir modified: /mnt/lv_ai_1_dante/ml/models/wav2vec2-1b-nst/wandb/run-20221025_092127-29xi74uq/files/output.log -2022-10-25 19:42:26,294 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:42:26,294 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:42:36,373 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:42:41,520 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:42:41,521 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:42:56,749 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:42:56,750 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:43:06,458 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:43:11,978 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:43:11,979 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:43:27,225 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:43:27,225 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:43:36,543 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:43:42,457 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:43:42,458 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:43:57,689 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:43:57,690 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:44:06,625 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:44:12,914 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:44:12,915 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:44:28,147 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:44:28,148 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:44:36,711 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:44:43,377 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:44:43,377 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:44:58,606 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:44:58,606 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:45:06,798 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:45:13,833 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:45:13,833 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:45:29,059 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:45:29,059 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:45:36,882 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats -2022-10-25 19:45:44,285 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:45:44,286 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:45:59,513 DEBUG HandlerThread:2194005 [handler.py:handle_request():141] handle_request: stop_status -2022-10-25 19:45:59,513 DEBUG SenderThread:2194005 [sender.py:send_request():246] send_request: stop_status -2022-10-25 19:46:06,969 DEBUG SenderThread:2194005 [sender.py:send():232] send: stats +version https://git-lfs.github.com/spec/v1 +oid sha256:d1f0f505e44d7c28f6208e4b95c48628f0454cf4ec5f9f738aecfce7b55aea56 +size 10978959